warn-scraper 1.2.154.dev0__py3-none-any.whl → 1.2.155.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- warn/pdfrodent/pdfrodent.py +10 -7
- warn/scrapers/la.py +120 -396
- warn/utils.py +1 -1
- {warn_scraper-1.2.154.dev0.dist-info → warn_scraper-1.2.155.dev0.dist-info}/METADATA +1 -1
- {warn_scraper-1.2.154.dev0.dist-info → warn_scraper-1.2.155.dev0.dist-info}/RECORD +9 -9
- {warn_scraper-1.2.154.dev0.dist-info → warn_scraper-1.2.155.dev0.dist-info}/WHEEL +0 -0
- {warn_scraper-1.2.154.dev0.dist-info → warn_scraper-1.2.155.dev0.dist-info}/entry_points.txt +0 -0
- {warn_scraper-1.2.154.dev0.dist-info → warn_scraper-1.2.155.dev0.dist-info}/licenses/LICENSE +0 -0
- {warn_scraper-1.2.154.dev0.dist-info → warn_scraper-1.2.155.dev0.dist-info}/top_level.txt +0 -0
warn/pdfrodent/pdfrodent.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import json
|
|
2
1
|
import logging
|
|
3
2
|
import re
|
|
3
|
+
from pathlib import Path
|
|
4
4
|
|
|
5
5
|
import camelot # pip install camelot-py==1.0.9 for now
|
|
6
6
|
|
|
@@ -178,7 +178,7 @@ We probably want code that tells us what PDF this is pulled from, on which row.
|
|
|
178
178
|
"""
|
|
179
179
|
|
|
180
180
|
|
|
181
|
-
def parse_pdf(pdffile: str, field_fixes: dict | None = None):
|
|
181
|
+
def parse_pdf(pdffile: str | Path, field_fixes: dict | None = None):
|
|
182
182
|
"""Parse a PDF file to extract data from tables.
|
|
183
183
|
|
|
184
184
|
Args:
|
|
@@ -227,7 +227,7 @@ def parse_pdf(pdffile: str, field_fixes: dict | None = None):
|
|
|
227
227
|
patchedheaders.append(field_fixes[item])
|
|
228
228
|
else:
|
|
229
229
|
logger.debug(
|
|
230
|
-
f"New header type found: {item}, not in {' '.join(sorted(list(field_fixes.keys())))}"
|
|
230
|
+
f"New header type found: {item}, not in field_fixes: '{' '.join(sorted(list(field_fixes.keys())))}'"
|
|
231
231
|
)
|
|
232
232
|
patchedheaders.append(item)
|
|
233
233
|
orphanholder = {
|
|
@@ -257,7 +257,7 @@ def parse_pdf(pdffile: str, field_fixes: dict | None = None):
|
|
|
257
257
|
patchedheaders.append(field_fixes[item])
|
|
258
258
|
else:
|
|
259
259
|
logger.debug(
|
|
260
|
-
f"New header type found: {item}, not in {' '.join(sorted(list(field_fixes.keys())))}"
|
|
260
|
+
f"New header type found: {item}, not in field_fixes: '{' '.join(sorted(list(field_fixes.keys())))}'"
|
|
261
261
|
)
|
|
262
262
|
patchedheaders.append(item)
|
|
263
263
|
headerfirst = patchedheaders
|
|
@@ -286,7 +286,7 @@ def parse_pdf(pdffile: str, field_fixes: dict | None = None):
|
|
|
286
286
|
isheader = False
|
|
287
287
|
orphanedheader = False
|
|
288
288
|
|
|
289
|
-
else: #
|
|
289
|
+
else: # seendata
|
|
290
290
|
if isheader: # Supplement to a header on a latter page
|
|
291
291
|
filerowholder.append(
|
|
292
292
|
"\tMostly empty row, seems to be appending to a header"
|
|
@@ -325,6 +325,7 @@ def parse_pdf(pdffile: str, field_fixes: dict | None = None):
|
|
|
325
325
|
locallist[-1][
|
|
326
326
|
fieldname
|
|
327
327
|
] = cleancell # Add it to the previous line
|
|
328
|
+
locallist[-1]["_int_raw_fields"].extend(row)
|
|
328
329
|
isheader = False
|
|
329
330
|
|
|
330
331
|
else:
|
|
@@ -341,16 +342,18 @@ def parse_pdf(pdffile: str, field_fixes: dict | None = None):
|
|
|
341
342
|
for cellindex, cell in enumerate(row):
|
|
342
343
|
line[headerfirst[cellindex]] = clean_cell(cell)
|
|
343
344
|
filerowholder.append(f"\t\t{line}")
|
|
345
|
+
line["_int_raw_fields"] = row
|
|
344
346
|
locallist.append(line)
|
|
345
347
|
|
|
346
348
|
report = table.parsing_report
|
|
347
349
|
|
|
348
350
|
for lineindex, line in enumerate(locallist):
|
|
349
351
|
line["_int_accuracy"] = report["accuracy"]
|
|
350
|
-
line["_int_pdf_filename"] = pdffile.split("/")[-1].split("\\")[-1]
|
|
352
|
+
line["_int_pdf_filename"] = str(pdffile).split("/")[-1].split("\\")[-1]
|
|
351
353
|
line["_int_page"] = report["page"]
|
|
352
354
|
line["_int_table_number"] = report["order"]
|
|
353
|
-
line["_int_raw_fields"] =
|
|
355
|
+
# line["_int_raw_fields"] = row # HEY! Need to handle for supplemented lines
|
|
356
|
+
# line["_int_raw_fields"] = json.dumps(list(line.values()))
|
|
354
357
|
line["_int_data_items"] = count_data_items(line) # type: ignore
|
|
355
358
|
if "Event Number" in line:
|
|
356
359
|
line["Event Number"] = line["Event Number"].replace("\n", "")
|
warn/scrapers/la.py
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
|
+
import json
|
|
1
2
|
import logging
|
|
2
3
|
import os
|
|
3
4
|
import re
|
|
4
|
-
from datetime import datetime
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
|
|
7
|
-
import pdfplumber
|
|
8
7
|
from bs4 import BeautifulSoup
|
|
9
8
|
|
|
9
|
+
from warn.pdfrodent import pdfrodent as pdfrodent
|
|
10
|
+
|
|
10
11
|
from .. import utils
|
|
11
12
|
from ..cache import Cache
|
|
12
13
|
|
|
@@ -19,6 +20,8 @@ __source__ = {
|
|
|
19
20
|
|
|
20
21
|
logger = logging.getLogger(__name__)
|
|
21
22
|
|
|
23
|
+
want_debugging_file = True
|
|
24
|
+
|
|
22
25
|
|
|
23
26
|
def scrape(
|
|
24
27
|
data_dir: Path = utils.WARN_DATA_DIR,
|
|
@@ -41,6 +44,55 @@ def scrape(
|
|
|
41
44
|
base_url = "https://www.laworks.net/"
|
|
42
45
|
file_base = "Downloads_WFD"
|
|
43
46
|
|
|
47
|
+
headerfixes = {
|
|
48
|
+
"Company Name": "company_original",
|
|
49
|
+
"Notice Date": "date_notice",
|
|
50
|
+
"Layoff Date": "date_action",
|
|
51
|
+
"Employees Affected": "affected",
|
|
52
|
+
"supplement_0": "notes",
|
|
53
|
+
"Address": "address_original",
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
ignore = """
|
|
57
|
+
# First handle the older files, if needed
|
|
58
|
+
pdffiles = sorted(cache.files(subdir="la/", glob_pattern="*.pdf"))
|
|
59
|
+
historicalneeded = False
|
|
60
|
+
historicalfiles = []
|
|
61
|
+
for myyear in range(2007, 2023 + 1):
|
|
62
|
+
targeturl = f"https://www.laworks.net/Downloads/WFD/WarnNotices{myyear}.pdf"
|
|
63
|
+
targetfilename = str(cache_dir / "la"/targeturl.split("/")[-1])
|
|
64
|
+
historicalfiles.append(targetfilename)
|
|
65
|
+
if targetfilename not in pdffiles:
|
|
66
|
+
logger.debug(f"Missing {targetfilename} from collection of {' ... '.join(pdffiles)}")
|
|
67
|
+
filebin, filehtml = utils.get_with_zyte(targeturl)
|
|
68
|
+
cache.write_binary(targetfilename, filebin)
|
|
69
|
+
historicalneeded = True
|
|
70
|
+
|
|
71
|
+
if historicalneeded:
|
|
72
|
+
logger.debug(f"Need to process historical records")
|
|
73
|
+
historicallist = []
|
|
74
|
+
historicaldebug = []
|
|
75
|
+
for historicalfile in historicalfiles:
|
|
76
|
+
locallist, localdebug = pdfrodent.parse_pdf(historicalfile, headerfixes)
|
|
77
|
+
historicallist.extend(locallist)
|
|
78
|
+
historicaldebug.extend(localdebug)
|
|
79
|
+
|
|
80
|
+
with open(Path(cache_dir) / "la/historical.json", "w") as outfile:
|
|
81
|
+
outfile.write(json.dumps(historicallist), indent=4*" ")
|
|
82
|
+
|
|
83
|
+
if want_debugging_file:
|
|
84
|
+
with open(Path(cache_dir) / "la/debugging-historical.txt", "w") as outfile:
|
|
85
|
+
for row in historicaldebug:
|
|
86
|
+
outfile.write(json.dumps(row) + "\r\n")
|
|
87
|
+
else:
|
|
88
|
+
logger.debug(f"No historical data missing.")
|
|
89
|
+
|
|
90
|
+
logger.debug(f"Fetching historical data")
|
|
91
|
+
with open(Path(cache_dir) / "la/historical.json", "r") as infile:
|
|
92
|
+
historicallist = json.loads(infile)
|
|
93
|
+
logger.debug(f"{len(historicallist):,} historical items imported.")
|
|
94
|
+
"""
|
|
95
|
+
|
|
44
96
|
# Download the root page
|
|
45
97
|
url = f"{base_url}Downloads/{file_base}.asp"
|
|
46
98
|
htmlbin, html = utils.get_with_zyte(url)
|
|
@@ -54,7 +106,8 @@ def scrape(
|
|
|
54
106
|
document = BeautifulSoup(html, "html.parser")
|
|
55
107
|
links = document.find_all("a")
|
|
56
108
|
|
|
57
|
-
|
|
109
|
+
masterlist = []
|
|
110
|
+
fulldebug = []
|
|
58
111
|
|
|
59
112
|
for link in links:
|
|
60
113
|
if "WARN Notices" in link.text:
|
|
@@ -68,405 +121,76 @@ def scrape(
|
|
|
68
121
|
|
|
69
122
|
# Process the PDF
|
|
70
123
|
logger.debug(f"Attempting to parse {pdf_path}")
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
# Insert a header row with clean column names.
|
|
75
|
-
# We are here assuming that the columns don't change between years
|
|
76
|
-
# and that one that contains "Employees Affected" will be clean.
|
|
77
|
-
output_rows = [list(filter(_is_clean_header, all_rows))[0]] + list(
|
|
78
|
-
filter(lambda row: not _is_header(row), all_rows)
|
|
79
|
-
)
|
|
80
|
-
|
|
81
|
-
# Write out to CSV
|
|
82
|
-
data_path = data_dir / f"{state_code}.csv"
|
|
83
|
-
utils.write_rows_to_csv(data_path, output_rows)
|
|
84
|
-
|
|
85
|
-
# Return the path
|
|
86
|
-
return data_path
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
def _append_contents_to_cells_in_row_above(rows: list, index: int, row: list) -> list:
|
|
90
|
-
"""
|
|
91
|
-
Append the contents of a row to the contents of the cell above it.
|
|
92
|
-
|
|
93
|
-
Keyword arguments:
|
|
94
|
-
index -- the index of the row
|
|
95
|
-
row -- the row to check
|
|
96
|
-
rows -- the rows to append to
|
|
97
|
-
"""
|
|
98
|
-
for column_index, cell in enumerate(row):
|
|
99
|
-
if _cell_above_exists(column_index, rows):
|
|
100
|
-
rows[len(rows) - 1][column_index].extend(cell)
|
|
101
|
-
return rows
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
def _append_contents_to_row_from_row_above(rows: list, index: int, row: list) -> list:
|
|
105
|
-
"""
|
|
106
|
-
Append the contents of preceding row to a largely empty row.
|
|
107
|
-
|
|
108
|
-
Keyword arguments:
|
|
109
|
-
index -- the index of the row
|
|
110
|
-
row -- the row to amend
|
|
111
|
-
rows -- list of rows that includes the final row to pull from
|
|
112
|
-
"""
|
|
113
|
-
for column_index, cell in enumerate(row):
|
|
114
|
-
if _cell_above_exists(column_index, rows):
|
|
115
|
-
if len(cell) == 0:
|
|
116
|
-
row[column_index] = rows[len(rows) - 1][column_index]
|
|
117
|
-
return row
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
def _cell_above_exists(column_index: int, rows: list) -> bool:
|
|
121
|
-
"""
|
|
122
|
-
Return True if the cell above the current cell exists.
|
|
123
|
-
|
|
124
|
-
Keyword arguments:
|
|
125
|
-
column_index -- the index of the column
|
|
126
|
-
rows -- the rows to check
|
|
127
|
-
"""
|
|
128
|
-
return column_index < len(rows[len(rows) - 1])
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
def _has_rows(rows: list) -> bool:
|
|
132
|
-
"""
|
|
133
|
-
Determine if the table has rows.
|
|
134
|
-
|
|
135
|
-
Keyword arguments:
|
|
136
|
-
rows -- the rows to check
|
|
137
|
-
|
|
138
|
-
Returns: True if the table has rows, False otherwise
|
|
139
|
-
"""
|
|
140
|
-
return len(rows) > 0
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
def _is_first(index: int) -> bool:
|
|
144
|
-
"""
|
|
145
|
-
Determine if a row is the first row in a table.
|
|
146
|
-
|
|
147
|
-
Keyword arguments:
|
|
148
|
-
index -- the index of the row
|
|
149
|
-
|
|
150
|
-
Returns: True if the row is the first row in a table, False otherwise
|
|
151
|
-
"""
|
|
152
|
-
return index == 0
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
def _is_mostly_empty(row: list) -> bool:
|
|
156
|
-
"""
|
|
157
|
-
Check if a row has few populated cells. Used to determine if carried over from a previous page.
|
|
158
|
-
|
|
159
|
-
Keyword arguments:
|
|
160
|
-
row -- the row to check
|
|
161
|
-
|
|
162
|
-
Returns: True if the row is mostly empty, False otherwise
|
|
163
|
-
"""
|
|
164
|
-
return len(list(filter(pdfplumber.utils.extract_text, row))) <= 2
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
def _process_pdf(pdf_path) -> list:
|
|
168
|
-
"""
|
|
169
|
-
Process a PDF file.
|
|
170
|
-
|
|
171
|
-
Keyword arguments:
|
|
172
|
-
pdf_path -- the path to the PDF file
|
|
173
|
-
|
|
174
|
-
Returns: a list of rows
|
|
175
|
-
"""
|
|
176
|
-
output_rows: list = []
|
|
177
|
-
|
|
178
|
-
with pdfplumber.open(pdf_path) as pdf:
|
|
179
|
-
for page in pdf.pages:
|
|
180
|
-
for table in page.debug_tablefinder().tables:
|
|
181
|
-
for index, row in enumerate(table.rows):
|
|
182
|
-
cells = row.cells
|
|
183
|
-
cells = [_extract_cell_chars(page, cell) for cell in cells]
|
|
184
|
-
|
|
185
|
-
# If the first row in a table is mostly empty,
|
|
186
|
-
# append its contents to the previous row
|
|
187
|
-
if (
|
|
188
|
-
_is_first(index)
|
|
189
|
-
and _is_mostly_empty(cells)
|
|
190
|
-
and _has_rows(output_rows)
|
|
191
|
-
):
|
|
192
|
-
output_rows = _append_contents_to_cells_in_row_above(
|
|
193
|
-
output_rows, index, cells
|
|
194
|
-
)
|
|
195
|
-
# Otherwise, if a row is mostly empty, pull data into blank cells and add current row
|
|
196
|
-
elif _is_mostly_empty(cells):
|
|
197
|
-
cells = _append_contents_to_row_from_row_above(
|
|
198
|
-
output_rows, index, cells
|
|
199
|
-
)
|
|
200
|
-
output_rows.append(cells)
|
|
201
|
-
# Otherwise, append the row
|
|
202
|
-
else:
|
|
203
|
-
output_rows.append(cells)
|
|
204
|
-
|
|
205
|
-
return _clean_rows(output_rows)
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
def _clean_rows(rows):
|
|
209
|
-
"""
|
|
210
|
-
Clean up rows.
|
|
211
|
-
|
|
212
|
-
Keyword arguments:
|
|
213
|
-
rows -- the rows to clean
|
|
214
|
-
|
|
215
|
-
Returns: the cleaned rows
|
|
216
|
-
"""
|
|
217
|
-
output_rows = []
|
|
218
|
-
|
|
219
|
-
for row in rows:
|
|
220
|
-
output_row = []
|
|
221
|
-
for column_index, chars in enumerate(row):
|
|
222
|
-
text = _clean_text(pdfplumber.utils.extract_text(chars))
|
|
223
|
-
|
|
224
|
-
# If we're on the first column, try to extract location and notes
|
|
225
|
-
if _is_first(column_index):
|
|
226
|
-
# Tries to extract a company name, appends it to the row
|
|
227
|
-
company_name = _extract_company_name(chars)
|
|
228
|
-
output_row.append(company_name)
|
|
229
|
-
remaining_text = text.replace(company_name, "")
|
|
230
|
-
|
|
231
|
-
# Tries to extract a note, typically UPDATE or WARN RESCINDED
|
|
232
|
-
note = _extract_note(chars).strip()
|
|
233
|
-
|
|
234
|
-
# Whatever is left is assumbed to be the location
|
|
235
|
-
location = remaining_text.strip().replace(note, "")
|
|
236
|
-
|
|
237
|
-
# Append the location and note to the row or headers for those
|
|
238
|
-
if _is_header(output_row):
|
|
239
|
-
output_row.append("Location")
|
|
240
|
-
output_row.append("Note")
|
|
241
|
-
else:
|
|
242
|
-
output_row.append(location)
|
|
243
|
-
output_row.append(note)
|
|
244
|
-
else:
|
|
245
|
-
# Appends the remaining text to the row
|
|
246
|
-
output_row.append(text)
|
|
247
|
-
|
|
248
|
-
output_rows.append(output_row)
|
|
249
|
-
|
|
250
|
-
return output_rows
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
def _extract_cell_chars(page, bbox):
|
|
254
|
-
"""
|
|
255
|
-
Extract the characters from a cell.
|
|
256
|
-
|
|
257
|
-
Keyword arguments:
|
|
258
|
-
page -- the page from which to extract the characters
|
|
259
|
-
bbox -- the bounding box of the cell
|
|
260
|
-
|
|
261
|
-
Returns: a list of characters
|
|
262
|
-
"""
|
|
263
|
-
# If the bounding box is empty, append an empty list
|
|
264
|
-
if bbox is None:
|
|
265
|
-
return []
|
|
266
|
-
|
|
267
|
-
# Expand the bounding box to ensure it encompasses the bottom line of text
|
|
268
|
-
vertical_threshold = 5
|
|
269
|
-
expanded_bbox = _vertically_expand_bounding_box(bbox, vertical_threshold)
|
|
270
|
-
|
|
271
|
-
# Get the characters from the cell
|
|
272
|
-
return page.within_bbox(expanded_bbox).chars
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
def _vertically_expand_bounding_box(bbox, increase):
|
|
276
|
-
"""
|
|
277
|
-
Expand the bounding box by a given amount in the vertical direction.
|
|
278
|
-
|
|
279
|
-
Keyword arguments:
|
|
280
|
-
bbox -- the bounding box to expand
|
|
281
|
-
increase -- the amount to expand the bounding box
|
|
282
|
-
|
|
283
|
-
Returns: the expanded bounding box
|
|
284
|
-
"""
|
|
285
|
-
return (
|
|
286
|
-
bbox[0],
|
|
287
|
-
bbox[1],
|
|
288
|
-
bbox[2],
|
|
289
|
-
bbox[3] + increase,
|
|
290
|
-
)
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
def _read_or_download(cache: Cache, prefix: str, url: str) -> Path:
|
|
294
|
-
"""
|
|
295
|
-
Read a file from the cache or downloads it.
|
|
296
|
-
|
|
297
|
-
Keyword arguments:
|
|
298
|
-
cache -- the cache to use
|
|
299
|
-
prefix -- the prefix to use for the cache key
|
|
300
|
-
url -- the URL to download
|
|
301
|
-
|
|
302
|
-
Returns: the path to the file
|
|
303
|
-
"""
|
|
304
|
-
file_name = os.path.basename(url)
|
|
305
|
-
cache_key = f"{prefix}/{file_name}"
|
|
306
|
-
|
|
307
|
-
exists = cache.exists(cache_key)
|
|
308
|
-
year = _extract_year(file_name)
|
|
309
|
-
current_year = datetime.now().year
|
|
310
|
-
|
|
311
|
-
# Form a file path so we can read from the cache
|
|
312
|
-
if exists and year < current_year - 1:
|
|
313
|
-
return cache.path / cache_key
|
|
314
|
-
|
|
315
|
-
return cache.download(cache_key, url)
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
def _extract_year(text: str) -> int:
|
|
319
|
-
"""
|
|
320
|
-
Extract the year from a PDF file name.
|
|
321
|
-
|
|
322
|
-
Keyword arguments:
|
|
323
|
-
text -- the text to extract the year from
|
|
324
|
-
|
|
325
|
-
Returns: the year
|
|
326
|
-
"""
|
|
327
|
-
year_pattern = re.compile(r"\d{4}", re.IGNORECASE)
|
|
328
|
-
year = re.search(year_pattern, text)
|
|
329
|
-
|
|
330
|
-
if year is not None:
|
|
331
|
-
return int(year.group(0))
|
|
332
|
-
else:
|
|
333
|
-
raise Exception(f"Could not extract year from {text}")
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
def _is_header(row: list) -> bool:
|
|
337
|
-
"""
|
|
338
|
-
Determine if a row is a header row.
|
|
339
|
-
|
|
340
|
-
Keyword arguments:
|
|
341
|
-
row -- the row to check
|
|
342
|
-
|
|
343
|
-
Returns: True if the row is a header row, False otherwise
|
|
344
|
-
"""
|
|
345
|
-
return row[0].strip().lower() == "company name"
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
def _is_clean_header(row: list) -> bool:
|
|
349
|
-
"""
|
|
350
|
-
Return true for a header with a clean column name.
|
|
351
|
-
|
|
352
|
-
Keyword arguments:
|
|
353
|
-
row -- the rows to check
|
|
354
|
-
|
|
355
|
-
Returns: true if the row is a clean header
|
|
356
|
-
"""
|
|
357
|
-
return _is_header(row) and "Employees Affected" in row
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
def _extract_note(chars) -> str:
|
|
361
|
-
"""
|
|
362
|
-
Extract a note from a PDF cell.
|
|
363
|
-
|
|
364
|
-
Keyword arguments:
|
|
365
|
-
chars -- the characters to extract the note from
|
|
124
|
+
locallist, localdebug = pdfrodent.parse_pdf(pdf_path, headerfixes)
|
|
125
|
+
masterlist.extend(locallist)
|
|
126
|
+
fulldebug.extend(localdebug)
|
|
366
127
|
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
# Split text into lines
|
|
372
|
-
lines = text.split("\n")
|
|
373
|
-
|
|
374
|
-
notes = []
|
|
375
|
-
|
|
376
|
-
for line in lines:
|
|
377
|
-
note_pattern = r"((UPDATE.*|WARN RESCINDED))+"
|
|
378
|
-
note = re.search(note_pattern, line, re.IGNORECASE)
|
|
379
|
-
if note:
|
|
380
|
-
notes.append(_clean_text(note[0]))
|
|
381
|
-
|
|
382
|
-
return " ".join(notes)
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
def _extract_company_name(chars) -> str:
|
|
386
|
-
"""
|
|
387
|
-
Extract the company name from a PDF cell.
|
|
388
|
-
|
|
389
|
-
Keyword arguments:
|
|
390
|
-
chars -- the characters to extract the company name from
|
|
128
|
+
# Earlier versions assumed headers were the same. Let's not do that.
|
|
129
|
+
# Identify all header elements, even in the ones we're about to remove.
|
|
130
|
+
allheaders = set()
|
|
391
131
|
|
|
392
|
-
|
|
132
|
+
ignore += """
|
|
133
|
+
logger.debug(f"Folding in historical data")
|
|
134
|
+
masterlist.extend(historicallist)
|
|
393
135
|
"""
|
|
394
|
-
|
|
136
|
+
if ignore:
|
|
137
|
+
logger.debug("Historical PDFs are not available to be incorporated.")
|
|
395
138
|
|
|
396
|
-
#
|
|
397
|
-
|
|
139
|
+
# Add a couple columns we're sort of missing
|
|
140
|
+
for rowindex, row in enumerate(masterlist):
|
|
141
|
+
if "address_original" in row:
|
|
142
|
+
masterlist[rowindex]["address"] = row["address_original"]
|
|
143
|
+
masterlist[rowindex]["company"] = row["company_original"]
|
|
144
|
+
# " ".join(row["_int_raw_fields"][0].split("\n")[0])
|
|
398
145
|
|
|
399
|
-
# We're assuming first line is always part of a company name
|
|
400
|
-
company_name = _clean_text(lines[0])
|
|
401
|
-
|
|
402
|
-
# Try to extract bold text in the cell
|
|
403
|
-
bold_text = _clean_text(_extract_bold_text(chars))
|
|
404
|
-
remaining_bold_text = bold_text.replace(company_name, "").strip()
|
|
405
|
-
|
|
406
|
-
# Loop through all but first and last lines
|
|
407
|
-
for index, line in enumerate(lines[1:-1]):
|
|
408
|
-
line_text = _clean_text(line)
|
|
409
|
-
|
|
410
|
-
# If line is bolded or doesn't match a pattern
|
|
411
|
-
# it's probably part of the company name
|
|
412
|
-
if remaining_bold_text.startswith(line_text) or (
|
|
413
|
-
bold_text == "" and index < 1 and not _is_location(line_text)
|
|
414
|
-
):
|
|
415
|
-
company_name += f" {line_text}"
|
|
416
|
-
|
|
417
|
-
remaining_bold_text = remaining_bold_text.replace(line_text, "").strip()
|
|
418
|
-
# The first time we hit a line that doesn't match our expectations,
|
|
419
|
-
# we assume we've reached the end of the company name
|
|
420
146
|
else:
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
return clean_text
|
|
147
|
+
firstcell = row["_int_raw_fields"][0]
|
|
148
|
+
# Let's assume that in most cases the address begins on a new line with a number.
|
|
149
|
+
# In those cases, stuff from before that will be the company name.
|
|
150
|
+
# Stuff beginning with the number but not the new line will be the address.
|
|
151
|
+
|
|
152
|
+
# In other cases, we'll have to assume the first line is the company name
|
|
153
|
+
# and stuff on subsequent lines is the address.
|
|
154
|
+
|
|
155
|
+
if re.findall(r"(\n\d+)", firstcell, re.MULTILINE):
|
|
156
|
+
mysplit = re.findall(r"(\n\d+)", firstcell, re.MULTILINE)[0].replace(
|
|
157
|
+
"\n", ""
|
|
158
|
+
)
|
|
159
|
+
masterlist[rowindex]["company"] = (
|
|
160
|
+
firstcell.split(mysplit)[0].strip().replace("\n", ", ")
|
|
161
|
+
)
|
|
162
|
+
masterlist[rowindex]["address"] = (
|
|
163
|
+
mysplit
|
|
164
|
+
+ " "
|
|
165
|
+
+ mysplit.join(firstcell.split(mysplit)[1:]).replace("\n", ", ")
|
|
166
|
+
)
|
|
167
|
+
else:
|
|
168
|
+
masterlist[rowindex]["company"] = row["_int_raw_fields"][0].split("\n")[
|
|
169
|
+
0
|
|
170
|
+
]
|
|
171
|
+
masterlist[rowindex]["address"] = ", ".join(
|
|
172
|
+
row["_int_raw_fields"][0].split("\n")[1:]
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
for row in masterlist:
|
|
176
|
+
for item in row:
|
|
177
|
+
allheaders.add(item)
|
|
178
|
+
text = ""
|
|
179
|
+
for item in sorted(allheaders):
|
|
180
|
+
text += f"\t\t'{item}': ,\n"
|
|
181
|
+
with open(Path(cache_dir) / "la/allheaders.txt", "w") as outfile:
|
|
182
|
+
outfile.write(text)
|
|
183
|
+
|
|
184
|
+
if want_debugging_file:
|
|
185
|
+
with open(Path(cache_dir) / "la/debugging.txt", "w") as outfile:
|
|
186
|
+
for row in fulldebug:
|
|
187
|
+
outfile.write(json.dumps(row) + "\r\n")
|
|
188
|
+
|
|
189
|
+
targetfilename = data_dir / f"{state_code}.csv"
|
|
190
|
+
logger.debug(f"Writing {len(masterlist):,} rows of data to {targetfilename}")
|
|
191
|
+
utils.write_disparate_dict_rows_to_csv(targetfilename, masterlist)
|
|
192
|
+
|
|
193
|
+
return targetfilename
|
|
470
194
|
|
|
471
195
|
|
|
472
196
|
if __name__ == "__main__":
|
warn/utils.py
CHANGED
|
@@ -144,7 +144,7 @@ def get_with_zyte(url, json_extras=None):
|
|
|
144
144
|
returnbin = returntext.encode("utf-8")
|
|
145
145
|
# returnbin = bin(returntext)
|
|
146
146
|
else:
|
|
147
|
-
returnbin: bytes = b64decode(api_response.json()["httpResponseBody"])
|
|
147
|
+
returnbin: bytes = bytes(b64decode(api_response.json()["httpResponseBody"]))
|
|
148
148
|
returntext: str = returnbin.decode("utf-8", errors="backslashreplace")
|
|
149
149
|
logger.debug(f"Fetched {url}")
|
|
150
150
|
return (returnbin, returntext)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: warn-scraper
|
|
3
|
-
Version: 1.2.
|
|
3
|
+
Version: 1.2.155.dev0
|
|
4
4
|
Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
|
|
5
5
|
Author-email: Big Local News <biglocalnews@stanford.edu>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -2,9 +2,9 @@ warn/__init__.py,sha256=A07JFY1TyaPtVIndBa7IvTk13DETqIkLgRdk0A-MCoE,85
|
|
|
2
2
|
warn/cache.py,sha256=QBSHycchvRTkOQfHptOtZeTYiPgLP383jS8MTiGln_c,5969
|
|
3
3
|
warn/cli.py,sha256=ZqyJwICdHFkn2hEgbArj_upbElR9-TSDlYDqyEGeexE,2019
|
|
4
4
|
warn/runner.py,sha256=oeGRybGwpnkQKlPzRMlKxhsDt1GN4PZoX-vUwrsPgos,1894
|
|
5
|
-
warn/utils.py,sha256
|
|
5
|
+
warn/utils.py,sha256=h1V5IISj9c2REtNnGctp0JqVjCi_Mi0r_FlvzNlrKb0,13093
|
|
6
6
|
warn/pdfrodent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
-
warn/pdfrodent/pdfrodent.py,sha256=
|
|
7
|
+
warn/pdfrodent/pdfrodent.py,sha256=S2LXkKOdP0Jqzcg10hzGijTu83TMS72DzZqQceslJ70,14914
|
|
8
8
|
warn/platforms/__init__.py,sha256=wIZRDf4tbTuC8oKM4ZrTAtwNgbtMQGzPXMwDYCFyrog,81
|
|
9
9
|
warn/platforms/job_center/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
10
|
warn/platforms/job_center/cache.py,sha256=yhA3sE46lNFg8vEewSoRYVByi0YSlkBiKm7qoSUiTdM,1868
|
|
@@ -29,7 +29,7 @@ warn/scrapers/il.py,sha256=sygdvsNuB_Gvu3o_HidtpSP4FLz0szKb1zEHqGxVtlI,1563
|
|
|
29
29
|
warn/scrapers/in.py,sha256=0hOCdbUyls3xX7Q1z5OB6yCrXJFpsu2CmvfwRh9NTfY,2202
|
|
30
30
|
warn/scrapers/ks.py,sha256=F_3biEMF7zgCX2XVuUACR74Vyzapta4SaM9SY3EuZCU,1266
|
|
31
31
|
warn/scrapers/ky.py,sha256=7kJTNOzxChyXlcyBImmdwwmrczYksU8XNxbhQ2owmJs,9688
|
|
32
|
-
warn/scrapers/la.py,sha256=
|
|
32
|
+
warn/scrapers/la.py,sha256=52i7ICYH0R7omFyu-kmKi_s-8Rhy0vaPjNIKlW_OMC8,6938
|
|
33
33
|
warn/scrapers/md.py,sha256=hwgxXQnhyBWm8qF1dvxIThAX1MkrZbXLwRI9inO5t8g,4060
|
|
34
34
|
warn/scrapers/me.py,sha256=q36F4yJ7hvZsLayA3uBS1romo4X3Qf-sEi2Y7LAQCi8,1172
|
|
35
35
|
warn/scrapers/mi.py,sha256=Ppyawp4nbzSBODuzDKeqnO9_9do5MFwK4Y_f3uc6blE,5846
|
|
@@ -54,9 +54,9 @@ warn/scrapers/va.py,sha256=7Nle7qL0VNPiE653XyaP9HQqSfuJFDRr2kEkjOqLvFM,11269
|
|
|
54
54
|
warn/scrapers/vt.py,sha256=d-bo4WK2hkrk4BhCCmLpEovcoZltlvdIUB6O0uaMx5A,1186
|
|
55
55
|
warn/scrapers/wa.py,sha256=UXdVtHZo_a-XfoiyOooTRfTb9W3PErSZdKca6SRORgs,4282
|
|
56
56
|
warn/scrapers/wi.py,sha256=ClEzXkwZbop0W4fkQgsb5oHAPUrb4luUPGV-jOKwkcg,4855
|
|
57
|
-
warn_scraper-1.2.
|
|
58
|
-
warn_scraper-1.2.
|
|
59
|
-
warn_scraper-1.2.
|
|
60
|
-
warn_scraper-1.2.
|
|
61
|
-
warn_scraper-1.2.
|
|
62
|
-
warn_scraper-1.2.
|
|
57
|
+
warn_scraper-1.2.155.dev0.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
|
|
58
|
+
warn_scraper-1.2.155.dev0.dist-info/METADATA,sha256=Vz4NRjIMPAx1mNm6337Va3PAEhD4Zd4VTNeiGBumCG0,1780
|
|
59
|
+
warn_scraper-1.2.155.dev0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
60
|
+
warn_scraper-1.2.155.dev0.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
|
|
61
|
+
warn_scraper-1.2.155.dev0.dist-info/top_level.txt,sha256=dZfms6N3kqVXufiPOo7YqOrAcUtYfNH_oyGvYUk9FB4,5
|
|
62
|
+
warn_scraper-1.2.155.dev0.dist-info/RECORD,,
|
|
File without changes
|
{warn_scraper-1.2.154.dev0.dist-info → warn_scraper-1.2.155.dev0.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{warn_scraper-1.2.154.dev0.dist-info → warn_scraper-1.2.155.dev0.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
|
File without changes
|