ShinySC 0.7.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ShinySC/__init__.py ADDED
@@ -0,0 +1,10 @@
1
+
2
+ from .main import describe
3
+ from .main import make_url
4
+ from .main import help
5
+ from .main import simple_metadata
6
+ from .main import full_metadata
7
+ from .main import search
8
+ from .main import update_list
9
+
10
+ __all__ = ["describe", "make_url","help","simple_metadata","full_metadata","search","update_list"]
ShinySC/main.py ADDED
@@ -0,0 +1,687 @@
1
+ import re
2
+ import json
3
+ import requests
4
+ import urllib.parse
5
+ from datetime import datetime
6
+
7
+ _cached_metadata = None
8
+ _cached_cube_list = None
9
+
10
+ _codes = requests.get('https://www150.statcan.gc.ca/t1/wds/rest/getCodeSets')
11
+ _codes = _codes.json()
12
+
13
+ def _fmt_id(id):
14
+
15
+ id = str(id)
16
+
17
+ if id.count('-') == 3:
18
+ a,b,c,d = id.split('-')
19
+ return a+b+c
20
+
21
+ if id.count('-') == 2:
22
+ a,b,c = id.split('-')
23
+ return a+b+c
24
+
25
+ if len(id) == 10:
26
+ return id[:8]
27
+
28
+ return id
29
+
30
+ def _vali_date(date_text):
31
+ try:
32
+ datetime.strptime(date_text, '%Y-%m-%d')
33
+ return True
34
+
35
+ except ValueError:
36
+ return False
37
+
38
+ def _cube_list(id='',lang='en'):
39
+ """
40
+ Retrieves information about a cube (table).
41
+
42
+ Args:
43
+ id (int or str): The product ID (PID)
44
+ lang (str): The language ('en' or 'fr')
45
+ """
46
+
47
+ global _cached_cube_list
48
+
49
+ try:
50
+ if _cached_cube_list == None:
51
+ url = "https://www150.statcan.gc.ca/t1/wds/rest/getAllCubesList"
52
+ resp = requests.get(url)
53
+ table = resp.json()
54
+ _cached_cube_list = table
55
+
56
+ else:
57
+ table = _cached_cube_list
58
+
59
+ except Exception as e:
60
+ print('Failed to find "getAllCubesList" on WDS')
61
+
62
+ if id == '':
63
+ return table
64
+
65
+ for i in table:
66
+ if i['productId'] == id: return i
67
+ else:
68
+ print(f'Table {id} not found.')
69
+
70
+ def _remove_lang(obj,language):
71
+
72
+ language = 'En' if language == 'en' else 'Fr'
73
+
74
+ if isinstance(obj, dict):
75
+
76
+ obj = {k: _remove_lang(v,language) for k, v in obj.items() if not k.endswith(language)}
77
+
78
+ for k in obj.keys():
79
+ if isinstance(obj[k],list):
80
+ if len(obj[k]) > 0:
81
+ if isinstance(obj[k][0],dict):
82
+ obj[k] = [_remove_lang(i,language) for i in obj[k]]
83
+ return obj
84
+
85
+ else:
86
+ return obj
87
+
88
+ def _search_json(obj: dict={}, term: str='', mode: str='AND'):
89
+ """
90
+ Recursively search JSON-like structures.
91
+ Keep entries if:
92
+ - key contains term
93
+ - OR value contains term
94
+ - OR nested match
95
+ - OR sibling key contains "code" when any match occurs in the same dict
96
+ """
97
+
98
+ mode = mode.upper()
99
+
100
+ if mode == "OR":
101
+ terms = [t.strip() for t in term.split(",") if t.strip()]
102
+ pattern = re.compile("|".join(map(re.escape, terms)), re.IGNORECASE | re.DOTALL)
103
+
104
+ if mode == "AND":
105
+ query = [t.strip() for t in term.split(",") if t.strip()]
106
+ regex = "".join(
107
+ rf"(?=.*(?:^|\W){re.escape(t)}(?:\W|$))"
108
+ for t in query
109
+ )
110
+ pattern = re.compile(regex, re.IGNORECASE | re.DOTALL)
111
+
112
+ # Case 1: dict
113
+ if isinstance(obj, dict):
114
+ matches = {} # matches found in this dict
115
+ matched_in_dict = False # flag so we can keep sibling "code" keys
116
+
117
+ # First pass: detect matches
118
+ child_results = {}
119
+
120
+ for k, v in obj.items():
121
+
122
+ key_match = bool(pattern.search(k))
123
+ value_match = isinstance(v, str) and bool(pattern.search(v))
124
+
125
+ nested = None
126
+
127
+ if isinstance(v, (dict, list)):
128
+ nested = _search_json(v, term, mode)
129
+
130
+ # If this entry matches directly or via nested
131
+ if key_match or value_match or nested:
132
+ matched_in_dict = True
133
+ child_results[k] = nested if nested is not None else v
134
+
135
+ # Second pass: if any match happened, also keep sibling "code" keys
136
+ if matched_in_dict:
137
+ for k, v in obj.items():
138
+ if "Code" in k and k not in child_results:
139
+ print(k,v)
140
+ child_results[k] = v
141
+
142
+ return child_results if child_results else None
143
+
144
+ # Case 2: list
145
+ elif isinstance(obj, list):
146
+ results = []
147
+ for item in obj:
148
+ nested = _search_json(item, term, mode)
149
+ #value_match = isinstance(item, str) and term in item
150
+ value_match = isinstance(item, str) and bool(pattern.search(item))
151
+
152
+ if nested is not None or value_match:
153
+ results.append(nested if nested is not None else item)
154
+
155
+ return results if results else None
156
+
157
+ # Case 3: primitive
158
+ else:
159
+ #if isinstance(obj, str) and term in obj:
160
+ print(obj)
161
+ try:
162
+ if isinstance(obj, str) and bool(re.search(obj)):
163
+ return obj
164
+ except:
165
+ pass
166
+
167
+ return None
168
+
169
+ def _parse_dim(x,full=True):
170
+
171
+ tmp = {}
172
+ sel = []
173
+
174
+ if full:
175
+ for i in x:
176
+ try:
177
+ tmp[i['dimensionNameEn']] = i['member']
178
+ j = []
179
+ for k in i['member']:
180
+ j.append(k['memberId'])
181
+ except:
182
+ tmp[i['dimensionNameEn']] = i['hasUom']
183
+
184
+ sel.append(j)
185
+ else:
186
+ pass
187
+
188
+ return tmp,sel
189
+
190
+ def _parse_filters(filters,id,lang='en'):
191
+ """
192
+ Takes user-defined filters as a dictionary and
193
+ transforms it to use in the StatCan URL.
194
+
195
+ Args:
196
+ filters (dict): User-defined filters
197
+ """
198
+ #Iterate over dict; put it in same order metadata has them;
199
+ #Then convert selected items into their numeric entries as nested list
200
+ #--> Convert dimensionName to dimensionPositionId; this lets us put in right order
201
+ #-->---> Convert members' memberNameEn(Fr) to memberId
202
+ #There will be some "easy" filters, like '503' to select all CMAs in geography
203
+ #...though I guess that would be applied after the table is downloaded
204
+
205
+ tmp = full_metadata(id)
206
+ dim = tmp['dimension']
207
+
208
+ new_filters = []
209
+
210
+ for i in range(0,len(dim)):
211
+
212
+ dim_name = dim[i][f'dimensionName{lang.capitalize()}']
213
+
214
+ if dim_name in filters:
215
+
216
+ selected_names = filters[dim_name]
217
+ selected_ids = []
218
+
219
+ for j in dim[i]['member']:
220
+
221
+ if j[f'memberName{lang.capitalize()}'] in selected_names:
222
+ selected_ids.append(j['memberId'])
223
+
224
+ new_filters.append(selected_ids)
225
+
226
+ else:
227
+ #Select all members if not specified in filters
228
+ selected_ids = []
229
+ for j in dim[i]['member']:
230
+ selected_ids.append(j['memberId'])
231
+ new_filters.append(selected_ids)
232
+
233
+ raw = json.dumps(new_filters, separators=(',',':'))
234
+ enc = urllib.parse.quote(raw)
235
+
236
+ return enc
237
+
238
+ def full_metadata(id, timeout=30, lang='en', special=[]):
239
+ """
240
+ Retrieves metadata for a cube (table) from Statistics Canada WDS in JSON format.
241
+
242
+ :param id (str): productId
243
+ :param timeout (int): Timeout in seconds for the HTTP request.
244
+ :param lang (str): which langauge you wish to get data in ('en'[default] or 'fr')
245
+ """
246
+
247
+ if (id != '') & ((len(str(id)) != 8) | (any(c.isalpha() for c in str(id)))): raise ValueError('productId must be 8 digits.')
248
+
249
+ endpoint = "https://www150.statcan.gc.ca/t1/wds/rest/getCubeMetadata"
250
+
251
+ # Prepare payload as list of dicts, per documentation example. :contentReference[oaicite:2]{index=2}
252
+ payload = [
253
+ {
254
+ "productId": str(id)
255
+ }
256
+ ]
257
+
258
+ headers = {
259
+ "Content-Type": "application/json"
260
+ }
261
+
262
+ try:
263
+ response = requests.post(endpoint, headers=headers, json=payload, timeout=timeout)
264
+ response.raise_for_status()
265
+ data = response.json()
266
+
267
+ # Check for status
268
+ if data[0]["status"] != "SUCCESS":
269
+ raise RuntimeError(f"Request returned non-SUCCESS status: {data}")
270
+
271
+ if lang == 'en':
272
+ return _remove_lang(data[0]['object'],'Fr')
273
+
274
+ elif lang == 'fr':
275
+ return _remove_lang(data[0]['object'],'En')
276
+
277
+ else:
278
+ return data[0]['object']
279
+
280
+ except requests.RequestException as e:
281
+ raise RuntimeError(f"HTTP request failed: {e}") from e
282
+
283
+ except json.JSONDecodeError as e:
284
+ raise RuntimeError(f"Failed to parse JSON response: {e}") from e
285
+
286
+ def simple_metadata(id, lang='en'):
287
+ """
288
+ Retrieves and displays simplified metadata for a cube (table).\n
289
+ This will only show productId, cubeTitle, cubeEndDate, cubeStartDate, and a simplified dimension.
290
+
291
+ :param id (str): productId
292
+ :param lang (str): which langauge you wish to get data in ('en'[default] or 'fr')
293
+ """
294
+
295
+ if (id != '') & ((len(str(id)) != 8) | (any(c.isalpha() for c in str(id)))): raise ValueError('productId must be 8 digits.')
296
+
297
+ meta = full_metadata(id,30,lang)
298
+
299
+ keep = ['productId',f'cubeTitle{lang.capitalize}','cubeEndDate','cubeStartDate','dimension']
300
+
301
+ obj = {k:v for k,v in meta.items() if k in keep}
302
+
303
+ return obj
304
+
305
+ def help():
306
+ print("""
307
+ ---------------------------------------------------------
308
+
309
+ describe(): provides key information about a table to help you build a custom query
310
+ make_url(): builds a custom URL to download data from StatCan WDS
311
+ list_tables (): ists all active tables available on StatCan WDS
312
+ help(): displays this help message
313
+ simple_metadata(): retrieves simplified metadata for a table
314
+ full_metadata(): retrieves full metadata for a table
315
+ search (): earches for tables matching your criteria
316
+ update_list(): checks for recently updated tables
317
+
318
+
319
+ If you already know the productId and filters for the table
320
+ you want, simply call:
321
+
322
+ url = ShinySC.make_url(id={id}, filters={...})
323
+
324
+ Addtionally, you can specify date ranges or number of recent periods:
325
+
326
+ url = ShinySC.make_url(id={id}, periods=5)
327
+ url = ShinySC.make_url(id={id}, start='2020-01-01',end='2022-01-01')
328
+
329
+ If you don't know what table you want, you can programmatically
330
+ search for tables that match your criteria using these criteria:
331
+
332
+ ShinySC.search(query,last_updated,dates,status)
333
+
334
+ query: comma-separated terms to search for in table names, descriptions, and surveys
335
+ last_updated: 'YYYY-MM-DD' to search tables updated on or after this date
336
+ dates: list of two strings [start_date,end_date] to filter tables by date range
337
+ status: 'active' or 'archived' tables (default is 'active')
338
+
339
+ Call ShinySC.describe(productId) to see available dimensionsthat can be used for filtering.
340
+
341
+ Call ShinySC.update_list(date='YYYY-MM-DD') to see recently updated tables for a specfiic date.
342
+ Add in a productId to see if that specific table was updated on that date.
343
+
344
+ ShinySC.update_list(id=35100003,date='2023-01-01')
345
+
346
+ This library makes use of these endpoints:
347
+ https://www150.statcan.gc.ca/t1/wds/rest/getCodeSets
348
+ https://www150.statcan.gc.ca/t1/wds/rest/getAllCubesList
349
+ https://www150.statcan.gc.ca/t1/wds/rest/getCubeMetadata
350
+ https://www150.statcan.gc.ca/t1/wds/rest/getChangedCubeList
351
+
352
+ ---------------------------------------------------------
353
+ """)
354
+
355
+ def describe(id, lang='en'):
356
+ """
357
+ Quickly get key information to help you build a custom query.
358
+
359
+ :param id (str): productId
360
+ :param lang (str): which langauge you wish to get data in ('en'[default] or 'fr')
361
+ """
362
+
363
+ if (id != '') & ((len(str(id)) != 8) | (any(c.isalpha() for c in str(id)))): raise ValueError('productId must be 8 digits.')
364
+
365
+ attributes = {}
366
+
367
+ md = full_metadata(id,30,lang)
368
+
369
+ attributes['name'] = md[f'cubeTitle{lang.capitalize()}']
370
+ attributes['dataPoints'] = md['nbDatapointsCube']
371
+ attributes['productId'] = id
372
+ attributes['status'] = 'Active' if md['archiveStatusCode'] == '2' else 'Archived'
373
+ attributes['dataDateRange'] = [md['cubeStartDate'],md['cubeEndDate']]
374
+ attributes['lastUpdated'] = md['releaseTime']
375
+ attributes['subject'] = md['subjectCode']
376
+ attributes['dimensions'] = [
377
+ {i[f'dimensionName{lang.capitalize()}']:[j[f'memberName{lang.capitalize()}'] for j in i['member']]} for i in md['dimension']
378
+ ]
379
+
380
+ return attributes
381
+
382
+ def make_url(id: int='',periods: int='',start: str='',end: str='',filters={},lang: str='en'):
383
+ """
384
+ Downloads a table from Statistics Canada using custom filters.
385
+ Default language is English ('en')
386
+ Returns a URL that can be downloaded with your favourite library.
387
+
388
+ Args:
389
+ :param id (int, str): productId
390
+ :param periods (int): number of recent periods you wish to download (default is 1)
391
+ :param start (str): desired start date for data series (YYYY-MM-DD)
392
+ :param end (str): desired end date for data series (YYYY-MM-DD)
393
+ :param full (bool): download the full table or not (default True)
394
+ :param filters (dict): filters you wish to apply ('attribute':[...],...)
395
+ :param lang (str): which langauge you wish to get data in ('en'[default] or 'fr')
396
+ """
397
+ global _cached_metadata
398
+ full = False
399
+
400
+ #Validate inputs
401
+ if (id != '') & ((len(str(id)) != 8) | (any(c.isalpha() for c in str(id)))): raise ValueError('productId must be 8 digits.')
402
+ if lang not in ['en','fr']: raise ValueError('Language must be "en" or "fr".')
403
+ if (periods != '') & (not isinstance(periods,int)): raise TypeError('Must be integer number of periods to download.')
404
+
405
+ if (start != '') & (not(_vali_date(start))): raise ValueError('Must be a valid date in the format YYYY-MM-DD.')
406
+ if (start > datetime.now().strftime('%Y-%m-%d')): raise ValueError('Invalid start date. Cannot be in the future.')
407
+ if (start > end) & (end != ''): raise ValueError('Start date cannot be after end date.')
408
+
409
+ if (end != '') & (not(_vali_date(end))): raise ValueError('Must be a valid date in the format YYYY-MM-DD.')
410
+ if (end < start) & (start != ''): raise ValueError('End date cannot be after start date.')
411
+
412
+ if not isinstance(filters,dict): raise ValueError('Filters must be a dictionary of filterName:[values,...].')
413
+
414
+ try:
415
+ if _cached_metadata == None:
416
+ md = full_metadata(id,30,lang)
417
+
418
+ else:
419
+ md = _cached_metadata
420
+ except:
421
+ print('Invalid productId.')
422
+ return ''
423
+
424
+ #tablename = md['cubeTitleEn']
425
+
426
+ archived = md['archiveStatusCode']
427
+ lastUpdated = md['releaseTime']
428
+
429
+ if archived == '1': print(f'ADVISORY: This table has been archived and does not get updated. Last updated: {lastUpdated}')
430
+
431
+ if filters == {} and not full and periods == '' and start == '' and end == '':
432
+ print('ADVISORY: No filters specified. Generating URL for the full table. Full tables can be very large.')
433
+ full = True
434
+
435
+ if full:
436
+ dim = md['dimension']
437
+ dim,selected = _parse_dim(dim,full)
438
+ raw = json.dumps(selected, separators=(',',':'))
439
+ filters = urllib.parse.quote(raw)
440
+
441
+ else:
442
+ filters = _parse_filters(filters,id,lang)
443
+
444
+ #Date filter stuff
445
+ if (periods == '') & (start == '') & (end == ''):
446
+ print('ADVISORY: No date range or periods specified.\nDefaulting to all periods, which may be a large amount of data.')
447
+ start = md['cubeStartDate']
448
+ end = md['cubeEndDate']
449
+
450
+ url = f'https://www150.statcan.gc.ca/t1/tbl1/en/dtl!downloadDbLoadingData-nonTraduit.action?pid={id}01&latestN={periods}&startDate={start}&endDate={end}&csvLocale={lang}&selectedMembers={filters}&checkedLevels='
451
+
452
+ return url
453
+
454
+ def search(query='',last_updated='',data_dates=[],status='active',mode='AND',lang='en'):
455
+ """
456
+ Docstring for find_tables
457
+
458
+ :param query (str): Comma-delimited terms to search for in table names, descriptions, and surveys
459
+ :param status (str): Active (default) or Archived
460
+ :param last_updated (str): Only return tables updated after this date (YYYY-MM-DD)
461
+ :param data_dates ([str,str]): Only return tables with data in this date range (YYYY-MM-DD)
462
+ :param mode (str): 'AND'[default] or 'OR' search mode for query terms
463
+ :param lang (str): Language ('en' or 'fr')
464
+ """
465
+
466
+ global _codes
467
+
468
+ lang = lang.lower()
469
+ mode = mode.upper()
470
+
471
+ #Validate inputs
472
+ if query == '': raise ValueError('Query cannot be empty.')
473
+ if query.count(',') >= 10: raise ValueError('Maximum of 10 search terms allowed.')
474
+ if query.count(',,') > 0: raise ValueError('Empty search terms are not allowed.')
475
+ if query.startswith(',') or query.endswith(','): raise ValueError('Empty search terms are not allowed.')
476
+
477
+ if lang not in ['en','fr']: raise ValueError('Language must be "en" or "fr".')
478
+ if status.lower() not in ['active','archived']: raise ValueError('Status must be "active" or "archived".')
479
+ if mode.upper() not in ['AND','OR']: raise ValueError('Mode must be "AND" or "OR".')
480
+
481
+ if (last_updated != '') & (not(_vali_date(last_updated))): raise ValueError('Must be a valid date in the format YYYY-MM-DD.')
482
+
483
+ if data_dates != []:
484
+
485
+ if len(data_dates) == 2:
486
+
487
+ start = data_dates[0]
488
+ end = data_dates[1]
489
+
490
+ if (start != '') & (not(_vali_date(start))): raise ValueError('Starting data must be a valid date in the format YYYY-MM-DD.')
491
+ if (start > datetime.now().strftime('%Y-%m-%d')): raise ValueError('Invalid start date. Cannot be in the future.')
492
+ if (start > end) & (end != ''): raise ValueError('Start date cannot be after end date.')
493
+
494
+ if (end != '') & (not(_vali_date(end))): raise ValueError('End data must be a valid date in the format YYYY-MM-DD.')
495
+ if (end < start) & (start != ''): raise ValueError('End date cannot be after start date.')
496
+
497
+ status = status.lower()
498
+ mode = mode.upper()
499
+ lang = lang.lower()
500
+
501
+ if status == 'active':
502
+ status = '2'
503
+ elif status == 'archived':
504
+ status = '1'
505
+ else:
506
+ raise ValueError('Status must be "active" or "archived".')
507
+
508
+ local_codes = _remove_lang(_codes,'Fr' if lang == 'en' else 'En')
509
+ local_codes = _search_json(local_codes,query,mode)
510
+
511
+ parsed_local_codes = {}
512
+
513
+ if local_codes != None:
514
+
515
+ for k in local_codes['object'].keys():
516
+ parsed_local_codes[k] = []
517
+ for i in local_codes['object'][k]:
518
+ try:
519
+ parsed_local_codes[k].append(i[f'{k}Code'])
520
+ except:
521
+ pass
522
+
523
+ tables = _cube_list()
524
+ filtered_tables = []
525
+
526
+ if mode == "OR":
527
+ terms = [t.strip() for t in query.split(",") if t.strip()]
528
+ pattern = re.compile("|".join(map(re.escape, terms)), re.IGNORECASE | re.DOTALL)
529
+
530
+ if mode == "AND":
531
+ query = [t.strip() for t in query.split(",") if t.strip()]
532
+ regex = "".join(
533
+ rf"(?=.*(?:^|\W){re.escape(t)}(?:\W|$))"
534
+ for t in query
535
+ )
536
+ pattern = re.compile(regex, re.IGNORECASE | re.DOTALL)
537
+
538
+ for t in tables:
539
+
540
+ cubeTitle = f'cubeTitle{lang.capitalize()}'
541
+ released = t['releaseTime'][:10]
542
+ startDate = t['cubeStartDate']
543
+ endDate = t['cubeEndDate']
544
+
545
+ if pattern.search(t[cubeTitle]):
546
+ if t['archived'] == status:
547
+
548
+ # prevent duplicates
549
+ existing_ids = {list(d.keys())[0] for d in filtered_tables}
550
+ if t['productId'] in existing_ids:
551
+ return # or continue if in a loop
552
+
553
+ if last_updated:
554
+ released_date = datetime.strptime(released, "%Y-%m-%d").date()
555
+ last_updated_date = datetime.strptime(last_updated, "%Y-%m-%d").date()
556
+
557
+ if released_date >= last_updated_date:
558
+ filtered_tables.append({t['productId']: t[cubeTitle]})
559
+ else:
560
+ filtered_tables.append({t['productId']: t[cubeTitle]})
561
+
562
+ for k in parsed_local_codes.keys():
563
+
564
+ if (t['productId'] not in filtered_tables) & (t['archived'] == status):
565
+ if (k == 'subject') & (len(parsed_local_codes[k]) > 0) & ('subjectCode' in t):
566
+ try:
567
+ filtered_tables.append({t['productId']:t[f'cubeTitle{lang.capitalize()}']}) if set(parsed_local_codes[k]).intersection(set(t['subjectCode'])) else None
568
+ except:
569
+ pass
570
+
571
+ if (k == 'survey') & (len(parsed_local_codes[k]) > 0) & (t['surveyCode'] != None):
572
+ try:
573
+ filtered_tables.append({t['productId']:t[f'cubeTitle{lang.capitalize()}']}) if set(parsed_local_codes[k]).intersection(set(t['surveyCode'])) else None
574
+ except:
575
+ pass
576
+
577
+ print(f'Found {len(filtered_tables)} tables matching search criteria, "{query}".\n')
578
+
579
+ return filtered_tables
580
+
581
+ def update_list(id='',date=''):
582
+ """
583
+ :param id (str): Product ID of the table to check
584
+ :param date (str): Date in the format YYYY-MM-DD to check for changes
585
+ """
586
+
587
+ #if they input just a date in the id field, treat it as date
588
+ if (id.count('-') == 2) & (date == '') & (_vali_date(id)):
589
+ date = id
590
+ id = ''
591
+
592
+ if (id != '') & ((len(str(id)) != 8) | (any(c.isalpha() for c in str(id)))): raise ValueError('productId must be 8 digits.')
593
+ if (date != '') & (not(_vali_date(date))): raise ValueError('Must be a valid date in the format YYYY-MM-DD.')
594
+
595
+ #If no date provided, use today's date
596
+ if date == '':
597
+ date = datetime.now().strftime('%Y-%m-%d')
598
+
599
+ url = f"https://www150.statcan.gc.ca/t1/wds/rest/getChangedCubeList/{date}"
600
+ resp = requests.get(url)
601
+ tables = resp.json()
602
+
603
+ if id != '':
604
+ for i in tables['object']:
605
+ if i['productId'] == int(id):
606
+ return True
607
+ else:
608
+ return False
609
+
610
+ elif id == '':
611
+ return tables['object']
612
+
613
+ def list_tables(lang='en'):
614
+
615
+ """
616
+ :param lang (str): Language of the table to return ('en' or 'fr')
617
+ """
618
+
619
+ lang = lang.lower()
620
+ if lang not in ['en','fr']: raise ValueError('Language must be "en" or "fr".')
621
+
622
+ url = "https://www150.statcan.gc.ca/t1/wds/rest/getAllCubesList"
623
+ resp = requests.get(url)
624
+ tables = resp.json()
625
+
626
+ df = pd.DataFrame(tables)
627
+
628
+ if lang == 'en':
629
+ df = df[[x for x in df.columns if x if 'Fr' not in x]]
630
+ elif lang == 'fr':
631
+ df = df[[x for x in df.columns if x if 'En' not in x]]
632
+
633
+ df = df[df['archived'] == '2']
634
+
635
+ archived_tables = df[df['archived'] == '1']
636
+ active_tables = df[df['archived'] == '2']
637
+
638
+ card_css = "border:1px solid black;margin:10px;padding:10px;background:white;max-width:200px;border-radius:3px;color:black;"
639
+ n = 2
640
+
641
+ count = 0
642
+
643
+ for p in tables:
644
+
645
+ id = p['productId']
646
+ endDate = datetime.fromisoformat(p['cubeEndDate'].replace("Z", "+00:00")).replace(tzinfo=None)
647
+
648
+ if p['archived'] == '2':
649
+
650
+ metadata = full_metadata(id)
651
+ num_dimensions = len(metadata['dimension'])
652
+ data = metadata['dimension']
653
+
654
+ tablename = p['cubeTitleEn']
655
+ dim = p['dimensions']
656
+ dim,selected = _parse_dim(dim)
657
+
658
+ raw = json.dumps(selected, separators=(',',':'))
659
+ enc = urllib.parse.quote(raw)
660
+
661
+ num_param = len(data)
662
+
663
+ attributes = {}
664
+ attributes['Table Name'] = tablename
665
+
666
+ for i in range(0,num_param):
667
+
668
+ sub_size = len(data[i]['member'])
669
+ top_name = data[i]['dimensionNameEn']
670
+
671
+ for j in range(0,sub_size):
672
+
673
+ #print(data[i]['member'][j])
674
+
675
+ name = data[i]['member'][j]['memberNameEn']
676
+ classification = data[i]['member'][j]['classificationCode']
677
+
678
+ if top_name in attributes:
679
+ attributes[top_name].append(name)
680
+ else:
681
+ attributes[top_name] = [name]
682
+
683
+ print(id,' - ',tablename,'\n',attributes)
684
+ #get_table(id=id,n=n,enc=enc)#region_type='503')
685
+
686
+ count += 1
687
+