readabs 0.2.4__tar.gz → 0.2.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {readabs-0.2.4 → readabs-0.2.6}/CHANGELOG.md +27 -0
- {readabs-0.2.4 → readabs-0.2.6}/PKG-INFO +41 -6
- {readabs-0.2.4 → readabs-0.2.6}/README.md +39 -4
- readabs-0.2.6/TODO.md +1 -0
- {readabs-0.2.4 → readabs-0.2.6}/docs/readabs/grab_abs_url.html +97 -99
- {readabs-0.2.4 → readabs-0.2.6}/docs/readabs/read_abs_cat.html +609 -607
- {readabs-0.2.4 → readabs-0.2.6}/docs/readabs/read_abs_series.html +203 -183
- {readabs-0.2.4 → readabs-0.2.6}/docs/readabs/recalibrate.html +270 -266
- {readabs-0.2.4 → readabs-0.2.6}/docs/readabs.html +289 -288
- readabs-0.2.6/docs/search.js +46 -0
- {readabs-0.2.4 → readabs-0.2.6}/pyproject.toml +2 -1
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/grab_abs_url.py +1 -1
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/read_abs_series.py +1 -1
- readabs-0.2.6/uv.lock +1634 -0
- readabs-0.2.4/TODO.md +0 -23
- readabs-0.2.4/docs/search.js +0 -46
- readabs-0.2.4/uv.lock +0 -1380
- {readabs-0.2.4 → readabs-0.2.6}/.gitignore +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/.pylintrc +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/.python-version +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/LICENSE +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/build-all.sh +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/build-docs.sh +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/docs/index.html +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/docs/readabs/abs_catalogue.html +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/docs/readabs/print_abs_catalogue.html +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/docs/readabs/rba_catalogue.html +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/docs/readabs/read_abs_by_desc.html +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/docs/readabs/read_rba_table.html +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/docs/readabs/search_abs_meta.html +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/docs/readabs/splice.html +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/.test-data/Qrtly-CPI-Time-series-spreadsheets-all.zip +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/__init__.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/abs_catalogue.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/abs_meta_data.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/datatype.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/download_cache.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/get_abs_links.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/lint-all.sh +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/print_abs_catalogue.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/py.typed +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/rba_catalogue.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/rba_meta_data.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/read_abs_by_desc.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/read_abs_cat.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/read_rba_table.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/read_support.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/recalibrate.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/search_abs_meta.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/splice.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/src/readabs/utilities.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/test/test.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/test/test_offline_fallback.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/test/test_url_retrieval.py +0 -0
- {readabs-0.2.4 → readabs-0.2.6}/uv-upgrade.sh +0 -0
|
@@ -1,3 +1,30 @@
|
|
|
1
|
+
Version 0.2.6 released 22-Aug-2026 (Canberra Australia)
|
|
2
|
+
|
|
3
|
+
- Docs and tooling only; no code or behavioural change.
|
|
4
|
+
- README: the two `select_and_splice()` CPI examples are now self-contained —
|
|
5
|
+
the first shows the `read_abs_cat("6401.0")` call and the `base` selector it
|
|
6
|
+
uses, and the second replaces the undefined `INDICATOR_URL` placeholder with
|
|
7
|
+
a named `indicator_url` variable.
|
|
8
|
+
- Lint: added `CPY001` (missing copyright notice) to the ruff ignore list in
|
|
9
|
+
`pyproject.toml`. It was the only rule firing under the aggressive `ALL`
|
|
10
|
+
ruleset, once per module, and this project does not carry copyright headers.
|
|
11
|
+
`ruff check src/readabs` now passes cleanly.
|
|
12
|
+
- Refreshed the locked dependencies in `uv.lock`.
|
|
13
|
+
|
|
14
|
+
Version 0.2.5 released 22-Jun-2026 (Canberra Australia)
|
|
15
|
+
|
|
16
|
+
- Docs only; no code or behavioural change.
|
|
17
|
+
- Updated stale 7-digit `6202001` Labour Force table names to the current
|
|
18
|
+
8-digit `62020001` in the README, the `read_abs_series()` docstring, and the
|
|
19
|
+
`grab_abs_url()` self-test.
|
|
20
|
+
- Documented that `search_abs_meta()` matches substrings by default
|
|
21
|
+
(`exact_match=True` for exact matches).
|
|
22
|
+
- Added a "How readabs gets data" section (spreadsheet conventions, not a stable
|
|
23
|
+
API; SDMX Data API is beta/partial), plus links to the `sdmxabs` companion
|
|
24
|
+
package and Matt Cowgill's R `readabs`.
|
|
25
|
+
|
|
26
|
+
---
|
|
27
|
+
|
|
1
28
|
Version 0.2.4 released 21-Jun-2026 (Canberra Australia)
|
|
2
29
|
|
|
3
30
|
- Offline fallback: when fresh data cannot be downloaded (for example, there is
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: readabs
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.6
|
|
4
4
|
Summary: Get ABS timeseries data in pandas DataFrames
|
|
5
5
|
Project-URL: Repository, https://github.com/bpalmer4/readabs
|
|
6
6
|
Project-URL: Homepage, https://github.com/bpalmer4/readabs
|
|
@@ -64,7 +64,7 @@ data, meta = ra.read_abs_cat("6202.0")
|
|
|
64
64
|
# meta is a DataFrame containing all series metadata
|
|
65
65
|
|
|
66
66
|
# Access a specific table
|
|
67
|
-
labour_force = data["
|
|
67
|
+
labour_force = data["62020001"]
|
|
68
68
|
print(labour_force.head())
|
|
69
69
|
```
|
|
70
70
|
|
|
@@ -114,6 +114,8 @@ wanted = {
|
|
|
114
114
|
series_dict, meta = ra.read_abs_by_desc(wanted)
|
|
115
115
|
```
|
|
116
116
|
|
|
117
|
+
By default, search terms are matched as substrings (via `.str.contains()`), so a partial description like `"Unemployment rate ; Persons ;"` will match the full `"Unemployment rate ; Persons ; Australia ;"`. Pass `exact_match=True` to `search_abs_meta()` to require an exact (`==`) match instead.
|
|
118
|
+
|
|
117
119
|
### RBA Data
|
|
118
120
|
|
|
119
121
|
```python
|
|
@@ -156,7 +158,7 @@ The `read_abs_cat()` function accepts several optional parameters for fine-tunin
|
|
|
156
158
|
```python
|
|
157
159
|
data, meta = ra.read_abs_cat(
|
|
158
160
|
"6202.0",
|
|
159
|
-
single_excel_only="
|
|
161
|
+
single_excel_only="62020001", # Only download one specific table (faster)
|
|
160
162
|
cache_only=True, # Use cached data only (offline mode)
|
|
161
163
|
verbose=True, # Print diagnostic messages
|
|
162
164
|
ignore_errors=True, # Continue if some files fail to download
|
|
@@ -172,7 +174,7 @@ data, meta = ra.read_abs_cat(
|
|
|
172
174
|
|
|
173
175
|
| Parameter | Description |
|
|
174
176
|
|-----------|-------------|
|
|
175
|
-
| `single_excel_only` | Download only the specified Excel file (e.g., "
|
|
177
|
+
| `single_excel_only` | Download only the specified Excel file (e.g., "62020001") |
|
|
176
178
|
| `selected_excel` | Tuple of Excel file names to download (e.g., `("62020001", "62020017")`). Must be a tuple, not a list. |
|
|
177
179
|
| `single_zip_only` | Download only the specified ZIP file |
|
|
178
180
|
| `cache_only` | Only use locally cached files, don't download |
|
|
@@ -227,6 +229,9 @@ genuine ambiguity rather than guessing), **or a bare ABS Series ID string**
|
|
|
227
229
|
series you want. The two forms mix freely across sources:
|
|
228
230
|
|
|
229
231
|
```python
|
|
232
|
+
cur, cmeta = ra.read_abs_cat("6401.0") # CPI catalogue
|
|
233
|
+
base = {"Index Numbers ; All groups CPI ; Australia ;": mc.did, "Index Numbers": mc.unit}
|
|
234
|
+
|
|
230
235
|
series, unit, report = ra.select_and_splice([
|
|
231
236
|
(cur, cmeta, base | {"Month": mc.freq}), # by description
|
|
232
237
|
(cur, cmeta, "A2325846C"), # by Series ID (quarterly All groups CPI)
|
|
@@ -243,7 +248,8 @@ rescaled across reference-period changes):
|
|
|
243
248
|
|
|
244
249
|
```python
|
|
245
250
|
cur, cmeta = ra.read_abs_cat("6401.0") # monthly + long quarterly
|
|
246
|
-
|
|
251
|
+
indicator_url = "https://www.abs.gov.au/statistics/.../6484.0.zip" # archived release page/zip
|
|
252
|
+
ind, imeta = ra.read_abs_cat("6484.0", url=indicator_url) # discontinued -> fetch by URL
|
|
247
253
|
|
|
248
254
|
base = {"Index Numbers ; All groups CPI ; Australia ;": mc.did, "Index Numbers": mc.unit}
|
|
249
255
|
series, unit, report = ra.select_and_splice(
|
|
@@ -392,6 +398,33 @@ Most ABS functions return a tuple:
|
|
|
392
398
|
|
|
393
399
|
DataFrames use pandas `PeriodIndex` with appropriate frequency (Monthly, Quarterly, Yearly).
|
|
394
400
|
|
|
401
|
+
## How readabs gets data (and why it can occasionally break)
|
|
402
|
+
|
|
403
|
+
There is no official, stable ABS API that exposes the full time series
|
|
404
|
+
collection. The ABS publishes its data as Excel/zip files linked from
|
|
405
|
+
statistics landing pages, following conventions that are consistent but
|
|
406
|
+
undocumented: predictable file names, an `Index` sheet carrying the metadata,
|
|
407
|
+
and `{table}---{sheet}` sheet naming. readabs works by following those
|
|
408
|
+
conventions — it fetches the landing page, finds the spreadsheet links,
|
|
409
|
+
downloads them, and parses the sheets.
|
|
410
|
+
|
|
411
|
+
This means readabs depends on ABS *practice*, not a contract. When the ABS
|
|
412
|
+
renames a table, restructures a page, or discontinues a series, a call may
|
|
413
|
+
fail or return less than expected — not because of a bug here, but because the
|
|
414
|
+
upstream habit changed. readabs is built to fail loudly when this happens, and
|
|
415
|
+
it caches downloads so transient outages don't block you. If something stops
|
|
416
|
+
working, check whether the ABS has changed that release, and use the `url=` and
|
|
417
|
+
`history=` parameters to reach archived data.
|
|
418
|
+
|
|
419
|
+
The ABS does run an SDMX/REST data API at `data.api.abs.gov.au`. As of 2025 the
|
|
420
|
+
ABS describes it as *beta*: it carries only a subset of what's published on the
|
|
421
|
+
ABS website (some dataflows are not uploaded at all), it lags the website
|
|
422
|
+
release, and it has no mechanism for retrieving earlier vintages of a revised
|
|
423
|
+
series. For the data it does cover, a companion package —
|
|
424
|
+
[`sdmxabs`](https://github.com/bpalmer4/sdmxabs) — wraps that API directly.
|
|
425
|
+
readabs instead targets the published spreadsheets, which remain the most
|
|
426
|
+
complete source and (via `history=`) allow access to earlier vintages.
|
|
427
|
+
|
|
395
428
|
## Documentation
|
|
396
429
|
|
|
397
430
|
Full API documentation is available in the `./docs` directory. Generate updated documentation with:
|
|
@@ -415,3 +448,5 @@ This project is open source. See the repository for license details.
|
|
|
415
448
|
|
|
416
449
|
- **Repository**: https://github.com/bpalmer4/readabs
|
|
417
450
|
- **PyPI**: https://pypi.org/project/readabs/
|
|
451
|
+
- **sdmxabs** (companion package for the ABS SDMX/REST data API): https://github.com/bpalmer4/sdmxabs
|
|
452
|
+
- **readabs for R** (a separate, like-named ABS package by Matt Cowgill): https://github.com/mattcowgill/readabs
|
|
@@ -42,7 +42,7 @@ data, meta = ra.read_abs_cat("6202.0")
|
|
|
42
42
|
# meta is a DataFrame containing all series metadata
|
|
43
43
|
|
|
44
44
|
# Access a specific table
|
|
45
|
-
labour_force = data["
|
|
45
|
+
labour_force = data["62020001"]
|
|
46
46
|
print(labour_force.head())
|
|
47
47
|
```
|
|
48
48
|
|
|
@@ -92,6 +92,8 @@ wanted = {
|
|
|
92
92
|
series_dict, meta = ra.read_abs_by_desc(wanted)
|
|
93
93
|
```
|
|
94
94
|
|
|
95
|
+
By default, search terms are matched as substrings (via `.str.contains()`), so a partial description like `"Unemployment rate ; Persons ;"` will match the full `"Unemployment rate ; Persons ; Australia ;"`. Pass `exact_match=True` to `search_abs_meta()` to require an exact (`==`) match instead.
|
|
96
|
+
|
|
95
97
|
### RBA Data
|
|
96
98
|
|
|
97
99
|
```python
|
|
@@ -134,7 +136,7 @@ The `read_abs_cat()` function accepts several optional parameters for fine-tunin
|
|
|
134
136
|
```python
|
|
135
137
|
data, meta = ra.read_abs_cat(
|
|
136
138
|
"6202.0",
|
|
137
|
-
single_excel_only="
|
|
139
|
+
single_excel_only="62020001", # Only download one specific table (faster)
|
|
138
140
|
cache_only=True, # Use cached data only (offline mode)
|
|
139
141
|
verbose=True, # Print diagnostic messages
|
|
140
142
|
ignore_errors=True, # Continue if some files fail to download
|
|
@@ -150,7 +152,7 @@ data, meta = ra.read_abs_cat(
|
|
|
150
152
|
|
|
151
153
|
| Parameter | Description |
|
|
152
154
|
|-----------|-------------|
|
|
153
|
-
| `single_excel_only` | Download only the specified Excel file (e.g., "
|
|
155
|
+
| `single_excel_only` | Download only the specified Excel file (e.g., "62020001") |
|
|
154
156
|
| `selected_excel` | Tuple of Excel file names to download (e.g., `("62020001", "62020017")`). Must be a tuple, not a list. |
|
|
155
157
|
| `single_zip_only` | Download only the specified ZIP file |
|
|
156
158
|
| `cache_only` | Only use locally cached files, don't download |
|
|
@@ -205,6 +207,9 @@ genuine ambiguity rather than guessing), **or a bare ABS Series ID string**
|
|
|
205
207
|
series you want. The two forms mix freely across sources:
|
|
206
208
|
|
|
207
209
|
```python
|
|
210
|
+
cur, cmeta = ra.read_abs_cat("6401.0") # CPI catalogue
|
|
211
|
+
base = {"Index Numbers ; All groups CPI ; Australia ;": mc.did, "Index Numbers": mc.unit}
|
|
212
|
+
|
|
208
213
|
series, unit, report = ra.select_and_splice([
|
|
209
214
|
(cur, cmeta, base | {"Month": mc.freq}), # by description
|
|
210
215
|
(cur, cmeta, "A2325846C"), # by Series ID (quarterly All groups CPI)
|
|
@@ -221,7 +226,8 @@ rescaled across reference-period changes):
|
|
|
221
226
|
|
|
222
227
|
```python
|
|
223
228
|
cur, cmeta = ra.read_abs_cat("6401.0") # monthly + long quarterly
|
|
224
|
-
|
|
229
|
+
indicator_url = "https://www.abs.gov.au/statistics/.../6484.0.zip" # archived release page/zip
|
|
230
|
+
ind, imeta = ra.read_abs_cat("6484.0", url=indicator_url) # discontinued -> fetch by URL
|
|
225
231
|
|
|
226
232
|
base = {"Index Numbers ; All groups CPI ; Australia ;": mc.did, "Index Numbers": mc.unit}
|
|
227
233
|
series, unit, report = ra.select_and_splice(
|
|
@@ -370,6 +376,33 @@ Most ABS functions return a tuple:
|
|
|
370
376
|
|
|
371
377
|
DataFrames use pandas `PeriodIndex` with appropriate frequency (Monthly, Quarterly, Yearly).
|
|
372
378
|
|
|
379
|
+
## How readabs gets data (and why it can occasionally break)
|
|
380
|
+
|
|
381
|
+
There is no official, stable ABS API that exposes the full time series
|
|
382
|
+
collection. The ABS publishes its data as Excel/zip files linked from
|
|
383
|
+
statistics landing pages, following conventions that are consistent but
|
|
384
|
+
undocumented: predictable file names, an `Index` sheet carrying the metadata,
|
|
385
|
+
and `{table}---{sheet}` sheet naming. readabs works by following those
|
|
386
|
+
conventions — it fetches the landing page, finds the spreadsheet links,
|
|
387
|
+
downloads them, and parses the sheets.
|
|
388
|
+
|
|
389
|
+
This means readabs depends on ABS *practice*, not a contract. When the ABS
|
|
390
|
+
renames a table, restructures a page, or discontinues a series, a call may
|
|
391
|
+
fail or return less than expected — not because of a bug here, but because the
|
|
392
|
+
upstream habit changed. readabs is built to fail loudly when this happens, and
|
|
393
|
+
it caches downloads so transient outages don't block you. If something stops
|
|
394
|
+
working, check whether the ABS has changed that release, and use the `url=` and
|
|
395
|
+
`history=` parameters to reach archived data.
|
|
396
|
+
|
|
397
|
+
The ABS does run an SDMX/REST data API at `data.api.abs.gov.au`. As of 2025 the
|
|
398
|
+
ABS describes it as *beta*: it carries only a subset of what's published on the
|
|
399
|
+
ABS website (some dataflows are not uploaded at all), it lags the website
|
|
400
|
+
release, and it has no mechanism for retrieving earlier vintages of a revised
|
|
401
|
+
series. For the data it does cover, a companion package —
|
|
402
|
+
[`sdmxabs`](https://github.com/bpalmer4/sdmxabs) — wraps that API directly.
|
|
403
|
+
readabs instead targets the published spreadsheets, which remain the most
|
|
404
|
+
complete source and (via `history=`) allow access to earlier vintages.
|
|
405
|
+
|
|
373
406
|
## Documentation
|
|
374
407
|
|
|
375
408
|
Full API documentation is available in the `./docs` directory. Generate updated documentation with:
|
|
@@ -393,3 +426,5 @@ This project is open source. See the repository for license details.
|
|
|
393
426
|
|
|
394
427
|
- **Repository**: https://github.com/bpalmer4/readabs
|
|
395
428
|
- **PyPI**: https://pypi.org/project/readabs/
|
|
429
|
+
- **sdmxabs** (companion package for the ABS SDMX/REST data API): https://github.com/bpalmer4/sdmxabs
|
|
430
|
+
- **readabs for R** (a separate, like-named ABS package by Matt Cowgill): https://github.com/mattcowgill/readabs
|
readabs-0.2.6/TODO.md
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# TODO
|
|
@@ -429,108 +429,106 @@
|
|
|
429
429
|
</span><span id="L-356"><a href="#L-356"><span class="linenos">356</span></a> <span class="c1"># iterate over the sheets in the Excel file</span>
|
|
430
430
|
</span><span id="L-357"><a href="#L-357"><span class="linenos">357</span></a> <span class="k">for</span> <span class="n">sheet_name</span> <span class="ow">in</span> <span class="n">excel</span><span class="o">.</span><span class="n">sheet_names</span><span class="p">:</span>
|
|
431
431
|
</span><span id="L-358"><a href="#L-358"><span class="linenos">358</span></a> <span class="c1"># grab and go - no treatment of the data</span>
|
|
432
|
-
</span><span id="L-359"><a href="#L-359"><span class="linenos">359</span></a> <span class="n">sheet_data</span> <span class="o">=</span> <span class="n">
|
|
433
|
-
</span><span id="L-360"><a href="#L-360"><span class="linenos">360</span></a>
|
|
434
|
-
</span><span id="L-361"><a href="#L-361"><span class="linenos">361</span></a>
|
|
435
|
-
</span><span id="L-362"><a href="#L-362"><span class="linenos">362</span></a>
|
|
436
|
-
</span><span id="L-363"><a href="#L-363"><span class="linenos">363</span></a> <span class="k">
|
|
437
|
-
</span><span id="L-364"><a href="#L-364"><span class="linenos">364</span></a>
|
|
438
|
-
</span><span id="L-365"><a href="#L-365"><span class="linenos">365</span></a>
|
|
439
|
-
</span><span id="L-366"><a href="#L-366"><span class="linenos">366</span></a>
|
|
440
|
-
</span><span id="L-367"><a href="#L-367"><span class="linenos">367</span></a>
|
|
441
|
-
</span><span id="L-368"><a href="#L-368"><span class="linenos">368</span></a>
|
|
442
|
-
</span><span id="L-369"><a href="#L-369"><span class="linenos">369</span></a>
|
|
443
|
-
</span><span id="L-370"><a href="#L-370"><span class="linenos">370</span></a>
|
|
444
|
-
</span><span id="L-371"><a href="#L-371"><span class="linenos">371</span></a>
|
|
445
|
-
</span><span id="L-372"><a href="#L-372"><span class="linenos">372</span></a
|
|
446
|
-
</span><span id="L-373"><a href="#L-373"><span class="linenos">373</span></a> <span class="
|
|
447
|
-
</span><span id="L-374"><a href="#L-374"><span class="linenos">374</span></a>
|
|
448
|
-
</span><span id="L-375"><a href="#L-375"><span class="linenos">375</span></a
|
|
449
|
-
</span><span id="L-376"><a href="#L-376"><span class="linenos">376</span></a
|
|
450
|
-
</span><span id="L-377"><a href="#L-377"><span class="linenos">377</span></a><span class="
|
|
451
|
-
</span><span id="L-378"><a href="#L-378"><span class="linenos">378</span></a>
|
|
452
|
-
</span><span id="L-379"><a href="#L-379"><span class="linenos">379</span></a><span class="sd">
|
|
453
|
-
</span><span id="L-380"><a href="#L-380"><span class="linenos">380</span></a><span class="sd">
|
|
454
|
-
</span><span id="L-381"><a href="#L-381"><span class="linenos">381</span></a
|
|
455
|
-
</span><span id="L-382"><a href="#L-382"><span class="linenos">382</span></a
|
|
456
|
-
</span><span id="L-383"><a href="#L-383"><span class="linenos">383</span></a> <span class="
|
|
457
|
-
</span><span id="L-384"><a href="#L-384"><span class="linenos">384</span></a>
|
|
458
|
-
</span><span id="L-385"><a href="#L-385"><span class="linenos">385</span></a>
|
|
459
|
-
</span><span id="L-386"><a href="#L-386"><span class="linenos">386</span></a>
|
|
460
|
-
</span><span id="L-387"><a href="#L-387"><span class="linenos">387</span></a>
|
|
432
|
+
</span><span id="L-359"><a href="#L-359"><span class="linenos">359</span></a> <span class="n">sheet_data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_excel</span><span class="p">(</span><span class="n">excel</span><span class="p">,</span> <span class="n">sheet_name</span><span class="o">=</span><span class="n">sheet_name</span><span class="p">)</span>
|
|
433
|
+
</span><span id="L-360"><a href="#L-360"><span class="linenos">360</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">sheet_data</span><span class="p">)</span> <span class="o">==</span> <span class="n">EMPTY_BYTES_LENGTH</span><span class="p">:</span>
|
|
434
|
+
</span><span id="L-361"><a href="#L-361"><span class="linenos">361</span></a> <span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
|
|
435
|
+
</span><span id="L-362"><a href="#L-362"><span class="linenos">362</span></a> <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"_add_excel_bytes(): sheet </span><span class="si">{</span><span class="n">sheet_name</span><span class="si">}</span><span class="s2"> in </span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2"> is empty."</span><span class="p">)</span>
|
|
436
|
+
</span><span id="L-363"><a href="#L-363"><span class="linenos">363</span></a> <span class="k">continue</span>
|
|
437
|
+
</span><span id="L-364"><a href="#L-364"><span class="linenos">364</span></a> <span class="n">abs_dict</span><span class="p">[</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">name</span><span class="si">}{</span><span class="n">HYPHEN</span><span class="si">}{</span><span class="n">sheet_name</span><span class="si">}</span><span class="s2">"</span><span class="p">]</span> <span class="o">=</span> <span class="n">sheet_data</span>
|
|
438
|
+
</span><span id="L-365"><a href="#L-365"><span class="linenos">365</span></a>
|
|
439
|
+
</span><span id="L-366"><a href="#L-366"><span class="linenos">366</span></a> <span class="c1"># return the dictionary of DataFrames</span>
|
|
440
|
+
</span><span id="L-367"><a href="#L-367"><span class="linenos">367</span></a> <span class="k">return</span> <span class="n">abs_dict</span>
|
|
441
|
+
</span><span id="L-368"><a href="#L-368"><span class="linenos">368</span></a>
|
|
442
|
+
</span><span id="L-369"><a href="#L-369"><span class="linenos">369</span></a>
|
|
443
|
+
</span><span id="L-370"><a href="#L-370"><span class="linenos">370</span></a><span class="k">def</span><span class="w"> </span><span class="nf">_add_excel</span><span class="p">(</span>
|
|
444
|
+
</span><span id="L-371"><a href="#L-371"><span class="linenos">371</span></a> <span class="n">abs_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">DataFrame</span><span class="p">],</span>
|
|
445
|
+
</span><span id="L-372"><a href="#L-372"><span class="linenos">372</span></a> <span class="n">link</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
|
446
|
+
</span><span id="L-373"><a href="#L-373"><span class="linenos">373</span></a> <span class="o">**</span><span class="n">args</span><span class="p">:</span> <span class="n">Any</span><span class="p">,</span> <span class="c1"># ReadArgs compatible</span>
|
|
447
|
+
</span><span id="L-374"><a href="#L-374"><span class="linenos">374</span></a><span class="p">)</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">DataFrame</span><span class="p">]:</span>
|
|
448
|
+
</span><span id="L-375"><a href="#L-375"><span class="linenos">375</span></a><span class="w"> </span><span class="sd">"""Read in an Excel file at the URL in the 'link' argument.</span>
|
|
449
|
+
</span><span id="L-376"><a href="#L-376"><span class="linenos">376</span></a>
|
|
450
|
+
</span><span id="L-377"><a href="#L-377"><span class="linenos">377</span></a><span class="sd"> Pass those bytes to _add_excel_bytes() to put the contents</span>
|
|
451
|
+
</span><span id="L-378"><a href="#L-378"><span class="linenos">378</span></a><span class="sd"> into the dictionary of DataFrames given by 'abs_dict'. When done,</span>
|
|
452
|
+
</span><span id="L-379"><a href="#L-379"><span class="linenos">379</span></a><span class="sd"> return the dictionary of DataFrames.</span>
|
|
453
|
+
</span><span id="L-380"><a href="#L-380"><span class="linenos">380</span></a><span class="sd"> """</span>
|
|
454
|
+
</span><span id="L-381"><a href="#L-381"><span class="linenos">381</span></a> <span class="n">name</span> <span class="o">=</span> <span class="n">get_table_name</span><span class="p">(</span><span class="n">link</span><span class="p">)</span>
|
|
455
|
+
</span><span id="L-382"><a href="#L-382"><span class="linenos">382</span></a>
|
|
456
|
+
</span><span id="L-383"><a href="#L-383"><span class="linenos">383</span></a> <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">abs_dict</span><span class="p">:</span>
|
|
457
|
+
</span><span id="L-384"><a href="#L-384"><span class="linenos">384</span></a> <span class="c1"># table already in the dictionary</span>
|
|
458
|
+
</span><span id="L-385"><a href="#L-385"><span class="linenos">385</span></a> <span class="k">return</span> <span class="n">abs_dict</span>
|
|
459
|
+
</span><span id="L-386"><a href="#L-386"><span class="linenos">386</span></a>
|
|
460
|
+
</span><span id="L-387"><a href="#L-387"><span class="linenos">387</span></a> <span class="n">raw_bytes</span> <span class="o">=</span> <span class="n">get_file</span><span class="p">(</span><span class="n">link</span><span class="p">,</span> <span class="o">**</span><span class="n">args</span><span class="p">)</span>
|
|
461
461
|
</span><span id="L-388"><a href="#L-388"><span class="linenos">388</span></a>
|
|
462
|
-
</span><span id="L-389"><a href="#L-389"><span class="linenos">389</span></a> <span class="
|
|
462
|
+
</span><span id="L-389"><a href="#L-389"><span class="linenos">389</span></a> <span class="k">return</span> <span class="n">_add_excel_bytes</span><span class="p">(</span><span class="n">abs_dict</span><span class="p">,</span> <span class="n">raw_bytes</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">args</span><span class="p">)</span>
|
|
463
463
|
</span><span id="L-390"><a href="#L-390"><span class="linenos">390</span></a>
|
|
464
|
-
</span><span id="L-391"><a href="#L-391"><span class="linenos">391</span></a>
|
|
465
|
-
</span><span id="L-392"><a href="#L-392"><span class="linenos">392</span></a>
|
|
466
|
-
</span><span id="L-393"><a href="#L-393"><span class="linenos">393</span></a>
|
|
467
|
-
</span><span id="L-394"><a href="#L-394"><span class="linenos">394</span></a
|
|
468
|
-
</span><span id="L-395"><a href="#L-395"><span class="linenos">395</span></a
|
|
469
|
-
</span><span id="L-396"><a href="#L-396"><span class="linenos">396</span></a>
|
|
470
|
-
</span><span id="L-397"><a href="#L-397"><span class="linenos">397</span></a>
|
|
471
|
-
</span><span id="L-398"><a href="#L-398"><span class="linenos">398</span></a><span class="w">
|
|
472
|
-
</span><span id="L-399"><a href="#L-399"><span class="linenos">399</span></a>
|
|
473
|
-
</span><span id="L-400"><a href="#L-400"><span class="linenos">400</span></a>
|
|
474
|
-
</span><span id="L-401"><a href="#L-401"><span class="linenos">401</span></a>
|
|
475
|
-
</span><span id="L-402"><a href="#L-402"><span class="linenos">402</span></a>
|
|
476
|
-
</span><span id="L-403"><a href="#L-403"><span class="linenos">403</span></a> <span class="
|
|
477
|
-
</span><span id="L-404"><a href="#L-404"><span class="linenos">404</span></a>
|
|
478
|
-
</span><span id="L-405"><a href="#L-405"><span class="linenos">405</span></a> <span class="k">
|
|
479
|
-
</span><span id="L-406"><a href="#L-406"><span class="linenos">406</span></a> <span class="nb">print</span><span class="p">(</span><span class="s2">"
|
|
480
|
-
</span><span id="L-407"><a href="#L-407"><span class="linenos">407</span></a>
|
|
481
|
-
</span><span id="L-408"><a href="#L-408"><span class="linenos">408</span></a>
|
|
482
|
-
</span><span id="L-409"><a href="#L-409"><span class="linenos">409</span></a> <span class="
|
|
483
|
-
</span><span id="L-410"><a href="#L-410"><span class="linenos">410</span></a>
|
|
484
|
-
</span><span id="L-411"><a href="#L-411"><span class="linenos">411</span></a>
|
|
485
|
-
</span><span id="L-412"><a href="#L-412"><span class="linenos">412</span></a>
|
|
486
|
-
</span><span id="L-413"><a href="#L-413"><span class="linenos">413</span></a>
|
|
487
|
-
</span><span id="L-414"><a href="#L-414"><span class="linenos">414</span></a>
|
|
488
|
-
</span><span id="L-415"><a href="#L-415"><span class="linenos">415</span></a>
|
|
489
|
-
</span><span id="L-416"><a href="#L-416"><span class="linenos">416</span></a> <span class="
|
|
490
|
-
</span><span id="L-417"><a href="#L-417"><span class="linenos">417</span></a>
|
|
491
|
-
</span><span id="L-418"><a href="#L-418"><span class="linenos">418</span></a> <span class="k">
|
|
492
|
-
</span><span id="L-419"><a href="#L-419"><span class="linenos">419</span></a> <span class="nb">print</span><span class="p">(</span><span class="s2">"
|
|
493
|
-
</span><span id="L-420"><a href="#L-420"><span class="linenos">420</span></a>
|
|
494
|
-
</span><span id="L-421"><a href="#L-421"><span class="linenos">421</span></a>
|
|
495
|
-
</span><span id="L-422"><a href="#L-422"><span class="linenos">422</span></a> <span class="
|
|
496
|
-
</span><span id="L-423"><a href="#L-423"><span class="linenos">423</span></a>
|
|
497
|
-
</span><span id="L-424"><a href="#L-424"><span class="linenos">424</span></a>
|
|
498
|
-
</span><span id="L-425"><a href="#L-425"><span class="linenos">425</span></a>
|
|
499
|
-
</span><span id="L-426"><a href="#L-426"><span class="linenos">426</span></a> <span class="
|
|
500
|
-
</span><span id="L-427"><a href="#L-427"><span class="linenos">427</span></a> <span class="n">
|
|
501
|
-
</span><span id="L-428"><a href="#L-428"><span class="linenos">428</span></a>
|
|
502
|
-
</span><span id="L-429"><a href="#L-429"><span class="linenos">429</span></a> <span class="
|
|
464
|
+
</span><span id="L-391"><a href="#L-391"><span class="linenos">391</span></a>
|
|
465
|
+
</span><span id="L-392"><a href="#L-392"><span class="linenos">392</span></a><span class="c1"># --- main ---</span>
|
|
466
|
+
</span><span id="L-393"><a href="#L-393"><span class="linenos">393</span></a><span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
|
|
467
|
+
</span><span id="L-394"><a href="#L-394"><span class="linenos">394</span></a>
|
|
468
|
+
</span><span id="L-395"><a href="#L-395"><span class="linenos">395</span></a> <span class="k">def</span><span class="w"> </span><span class="nf">simple_test</span><span class="p">()</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
|
469
|
+
</span><span id="L-396"><a href="#L-396"><span class="linenos">396</span></a><span class="w"> </span><span class="sd">"""Test the grab_abs_url and grab_abs_zip functions."""</span>
|
|
470
|
+
</span><span id="L-397"><a href="#L-397"><span class="linenos">397</span></a>
|
|
471
|
+
</span><span id="L-398"><a href="#L-398"><span class="linenos">398</span></a> <span class="k">def</span><span class="w"> </span><span class="nf">test</span><span class="p">(</span><span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> <span class="c1"># ReadArgs compatible</span>
|
|
472
|
+
</span><span id="L-399"><a href="#L-399"><span class="linenos">399</span></a> <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"TEST -- </span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
473
|
+
</span><span id="L-400"><a href="#L-400"><span class="linenos">400</span></a> <span class="k">try</span><span class="p">:</span>
|
|
474
|
+
</span><span id="L-401"><a href="#L-401"><span class="linenos">401</span></a> <span class="n">data_dict</span> <span class="o">=</span> <span class="n">grab_abs_url</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
|
475
|
+
</span><span id="L-402"><a href="#L-402"><span class="linenos">402</span></a> <span class="nb">print</span><span class="p">(</span><span class="s2">"---"</span><span class="p">)</span>
|
|
476
|
+
</span><span id="L-403"><a href="#L-403"><span class="linenos">403</span></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">data_dict</span><span class="p">:</span>
|
|
477
|
+
</span><span id="L-404"><a href="#L-404"><span class="linenos">404</span></a> <span class="nb">print</span><span class="p">(</span><span class="s2">"PROBLEM -- No data found."</span><span class="p">)</span>
|
|
478
|
+
</span><span id="L-405"><a href="#L-405"><span class="linenos">405</span></a> <span class="k">else</span><span class="p">:</span>
|
|
479
|
+
</span><span id="L-406"><a href="#L-406"><span class="linenos">406</span></a> <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"SUCCESS -- Found </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">data_dict</span><span class="p">)</span><span class="si">}</span><span class="s2"> datasets: </span><span class="si">{</span><span class="nb">list</span><span class="p">(</span><span class="n">data_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
480
|
+
</span><span id="L-407"><a href="#L-407"><span class="linenos">407</span></a> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
|
|
481
|
+
</span><span id="L-408"><a href="#L-408"><span class="linenos">408</span></a> <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"ERROR -- Test failed with exception: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
482
|
+
</span><span id="L-409"><a href="#L-409"><span class="linenos">409</span></a> <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Done.</span><span class="se">\n</span><span class="si">{</span><span class="s1">'='</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="mi">20</span><span class="si">}</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
|
483
|
+
</span><span id="L-410"><a href="#L-410"><span class="linenos">410</span></a>
|
|
484
|
+
</span><span id="L-411"><a href="#L-411"><span class="linenos">411</span></a> <span class="k">def</span><span class="w"> </span><span class="nf">test_zip</span><span class="p">(</span><span class="n">zip_path</span><span class="p">:</span> <span class="n">Path</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> <span class="c1"># ReadArgs compatible</span>
|
|
485
|
+
</span><span id="L-412"><a href="#L-412"><span class="linenos">412</span></a> <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"TEST -- grab_abs_zip() with </span><span class="si">{</span><span class="n">zip_path</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
486
|
+
</span><span id="L-413"><a href="#L-413"><span class="linenos">413</span></a> <span class="k">try</span><span class="p">:</span>
|
|
487
|
+
</span><span id="L-414"><a href="#L-414"><span class="linenos">414</span></a> <span class="n">data_dict</span> <span class="o">=</span> <span class="n">grab_abs_zip</span><span class="p">(</span><span class="n">zip_path</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
|
488
|
+
</span><span id="L-415"><a href="#L-415"><span class="linenos">415</span></a> <span class="nb">print</span><span class="p">(</span><span class="s2">"---"</span><span class="p">)</span>
|
|
489
|
+
</span><span id="L-416"><a href="#L-416"><span class="linenos">416</span></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">data_dict</span><span class="p">:</span>
|
|
490
|
+
</span><span id="L-417"><a href="#L-417"><span class="linenos">417</span></a> <span class="nb">print</span><span class="p">(</span><span class="s2">"PROBLEM -- No data found."</span><span class="p">)</span>
|
|
491
|
+
</span><span id="L-418"><a href="#L-418"><span class="linenos">418</span></a> <span class="k">else</span><span class="p">:</span>
|
|
492
|
+
</span><span id="L-419"><a href="#L-419"><span class="linenos">419</span></a> <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"SUCCESS -- Found </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">data_dict</span><span class="p">)</span><span class="si">}</span><span class="s2"> datasets: </span><span class="si">{</span><span class="nb">list</span><span class="p">(</span><span class="n">data_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
493
|
+
</span><span id="L-420"><a href="#L-420"><span class="linenos">420</span></a> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
|
|
494
|
+
</span><span id="L-421"><a href="#L-421"><span class="linenos">421</span></a> <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"ERROR -- Test failed with exception: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
495
|
+
</span><span id="L-422"><a href="#L-422"><span class="linenos">422</span></a> <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Done.</span><span class="se">\n</span><span class="si">{</span><span class="s1">'='</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="mi">20</span><span class="si">}</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
|
496
|
+
</span><span id="L-423"><a href="#L-423"><span class="linenos">423</span></a>
|
|
497
|
+
</span><span id="L-424"><a href="#L-424"><span class="linenos">424</span></a> <span class="c1"># 4 -- grab from url</span>
|
|
498
|
+
</span><span id="L-425"><a href="#L-425"><span class="linenos">425</span></a> <span class="n">_name</span> <span class="o">=</span> <span class="s2">".test-data/Qrtly-CPI-Time-series-spreadsheets-all.zip"</span>
|
|
499
|
+
</span><span id="L-426"><a href="#L-426"><span class="linenos">426</span></a> <span class="n">_zip_path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">_name</span><span class="p">)</span>
|
|
500
|
+
</span><span id="L-427"><a href="#L-427"><span class="linenos">427</span></a> <span class="n">test_zip</span><span class="p">(</span><span class="n">_zip_path</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
501
|
+
</span><span id="L-428"><a href="#L-428"><span class="linenos">428</span></a>
|
|
502
|
+
</span><span id="L-429"><a href="#L-429"><span class="linenos">429</span></a> <span class="c1"># --- various grab_abs_url() tests</span>
|
|
503
503
|
</span><span id="L-430"><a href="#L-430"><span class="linenos">430</span></a>
|
|
504
|
-
</span><span id="L-431"><a href="#L-431"><span class="linenos">431</span></a> <span class="
|
|
505
|
-
</span><span id="L-432"><a href="#L-432"><span class="linenos">432</span></a>
|
|
506
|
-
</span><span id="L-433"><a href="#L-433"><span class="linenos">433</span></a>
|
|
507
|
-
</span><span id="L-434"><a href="#L-434"><span class="linenos">434</span></a>
|
|
508
|
-
</span><span id="L-435"><a href="#L-435"><span class="linenos">435</span></a> <span class="n">
|
|
509
|
-
</span><span id="L-436"><a href="#L-436"><span class="linenos">436</span></a> <span class="n">
|
|
510
|
-
</span><span id="L-437"><a href="#L-437"><span class="linenos">437</span></a> <span class="n">
|
|
511
|
-
</span><span id="L-438"><a href="#L-438"><span class="linenos">438</span></a>
|
|
512
|
-
</span><span id="L-439"><a href="#L-439"><span class="linenos">439</span></a>
|
|
513
|
-
</span><span id="L-440"><a href="#L-440"><span class="linenos">440</span></a> <span class="
|
|
514
|
-
</span><span id="L-441"><a href="#L-441"><span class="linenos">441</span></a>
|
|
515
|
-
</span><span id="L-442"><a href="#L-442"><span class="linenos">442</span></a>
|
|
516
|
-
</span><span id="L-443"><a href="#L-443"><span class="linenos">443</span></a>
|
|
517
|
-
</span><span id="L-444"><a href="#L-444"><span class="linenos">444</span></a> <span class="n">
|
|
518
|
-
</span><span id="L-445"><a href="#L-445"><span class="linenos">445</span></a> <span class="n">
|
|
519
|
-
</span><span id="L-446"><a href="#L-446"><span class="linenos">446</span></a> <span class="n">
|
|
520
|
-
</span><span id="L-447"><a href="#L-447"><span class="linenos">447</span></a>
|
|
521
|
-
</span><span id="L-448"><a href="#L-448"><span class="linenos">448</span></a>
|
|
522
|
-
</span><span id="L-449"><a href="#L-449"><span class="linenos">449</span></a> <span class="
|
|
523
|
-
</span><span id="L-450"><a href="#L-450"><span class="linenos">450</span></a>
|
|
524
|
-
</span><span id="L-451"><a href="#L-451"><span class="linenos">451</span></a>
|
|
525
|
-
</span><span id="L-452"><a href="#L-452"><span class="linenos">452</span></a>
|
|
526
|
-
</span><span id="L-453"><a href="#L-453"><span class="linenos">453</span></a>
|
|
527
|
-
</span><span id="L-454"><a href="#L-454"><span class="linenos">454</span></a>
|
|
528
|
-
</span><span id="L-455"><a href="#L-455"><span class="linenos">455</span></a>
|
|
529
|
-
</span><span id="L-456"><a href="#L-456"><span class="linenos">456</span></a>
|
|
530
|
-
</span><span id="L-457"><a href="#L-457"><span class="linenos">457</span></a>
|
|
531
|
-
</span><span id="L-458"><a href="#L-458"><span class="linenos">458</span></a>
|
|
532
|
-
</span><span id="L-459"><a href="#L-459"><span class="linenos">459</span></a>
|
|
533
|
-
</span><span id="L-460"><a href="#L-460"><span class="linenos">460</span></a> <span class="n">simple_test</span><span class="p">()</span>
|
|
504
|
+
</span><span id="L-431"><a href="#L-431"><span class="linenos">431</span></a> <span class="n">name</span> <span class="o">=</span> <span class="s2">"1 -- grab a single zip file"</span>
|
|
505
|
+
</span><span id="L-432"><a href="#L-432"><span class="linenos">432</span></a> <span class="n">test</span><span class="p">(</span>
|
|
506
|
+
</span><span id="L-433"><a href="#L-433"><span class="linenos">433</span></a> <span class="n">name</span><span class="p">,</span>
|
|
507
|
+
</span><span id="L-434"><a href="#L-434"><span class="linenos">434</span></a> <span class="n">cat</span><span class="o">=</span><span class="s2">"6291.0.55.001"</span><span class="p">,</span>
|
|
508
|
+
</span><span id="L-435"><a href="#L-435"><span class="linenos">435</span></a> <span class="n">single_zip_only</span><span class="o">=</span><span class="s2">"p6291_all_quarterly_spreadsheets"</span><span class="p">,</span>
|
|
509
|
+
</span><span id="L-436"><a href="#L-436"><span class="linenos">436</span></a> <span class="n">get_zip</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
|
510
|
+
</span><span id="L-437"><a href="#L-437"><span class="linenos">437</span></a> <span class="n">verbose</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
|
511
|
+
</span><span id="L-438"><a href="#L-438"><span class="linenos">438</span></a> <span class="p">)</span>
|
|
512
|
+
</span><span id="L-439"><a href="#L-439"><span class="linenos">439</span></a>
|
|
513
|
+
</span><span id="L-440"><a href="#L-440"><span class="linenos">440</span></a> <span class="n">name</span> <span class="o">=</span> <span class="s2">"2 -- grab a single Excel file"</span>
|
|
514
|
+
</span><span id="L-441"><a href="#L-441"><span class="linenos">441</span></a> <span class="n">test</span><span class="p">(</span>
|
|
515
|
+
</span><span id="L-442"><a href="#L-442"><span class="linenos">442</span></a> <span class="n">name</span><span class="p">,</span>
|
|
516
|
+
</span><span id="L-443"><a href="#L-443"><span class="linenos">443</span></a> <span class="n">cat</span><span class="o">=</span><span class="s2">"6202.0"</span><span class="p">,</span>
|
|
517
|
+
</span><span id="L-444"><a href="#L-444"><span class="linenos">444</span></a> <span class="n">get_excel</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
|
518
|
+
</span><span id="L-445"><a href="#L-445"><span class="linenos">445</span></a> <span class="n">single_excel_only</span><span class="o">=</span><span class="s2">"62020001"</span><span class="p">,</span>
|
|
519
|
+
</span><span id="L-446"><a href="#L-446"><span class="linenos">446</span></a> <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
|
520
|
+
</span><span id="L-447"><a href="#L-447"><span class="linenos">447</span></a> <span class="p">)</span>
|
|
521
|
+
</span><span id="L-448"><a href="#L-448"><span class="linenos">448</span></a>
|
|
522
|
+
</span><span id="L-449"><a href="#L-449"><span class="linenos">449</span></a> <span class="c1"># 3 -- grab the whole shebang</span>
|
|
523
|
+
</span><span id="L-450"><a href="#L-450"><span class="linenos">450</span></a> <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span>
|
|
524
|
+
</span><span id="L-451"><a href="#L-451"><span class="linenos">451</span></a> <span class="s2">"https://www.abs.gov.au/statistics/labour/jobs/weekly-payroll-jobs/latest-release"</span><span class="p">,</span>
|
|
525
|
+
</span><span id="L-452"><a href="#L-452"><span class="linenos">452</span></a> <span class="s2">"https://www.abs.gov.au/statistics/people/population/national-state-and-territory-population/dec-2023"</span><span class="p">,</span>
|
|
526
|
+
</span><span id="L-453"><a href="#L-453"><span class="linenos">453</span></a> <span class="p">]</span>
|
|
527
|
+
</span><span id="L-454"><a href="#L-454"><span class="linenos">454</span></a> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">url_</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">urls</span><span class="p">):</span>
|
|
528
|
+
</span><span id="L-455"><a href="#L-455"><span class="linenos">455</span></a> <span class="n">name</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"3.</span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s2"> -- grab the whole shebang </span><span class="si">{</span><span class="n">url_</span><span class="si">}</span><span class="s2">"</span>
|
|
529
|
+
</span><span id="L-456"><a href="#L-456"><span class="linenos">456</span></a> <span class="n">test</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">url</span><span class="o">=</span><span class="n">url_</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
530
|
+
</span><span id="L-457"><a href="#L-457"><span class="linenos">457</span></a>
|
|
531
|
+
</span><span id="L-458"><a href="#L-458"><span class="linenos">458</span></a> <span class="n">simple_test</span><span class="p">()</span>
|
|
534
532
|
</span></pre></div>
|
|
535
533
|
|
|
536
534
|
|