irie 0.0.0__py3-none-any.whl → 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of irie might be problematic. Click here for more details.
- irie/__main__.py +1 -1
- irie/apps/config.py +2 -2
- irie/apps/documents/apps.py +1 -1
- irie/apps/documents/urls.py +1 -1
- irie/apps/evaluation/apps.py +2 -2
- irie/apps/evaluation/daemon.py +2 -2
- irie/apps/evaluation/identification.py +3 -3
- irie/apps/evaluation/models.py +2 -2
- irie/apps/evaluation/views.py +3 -3
- irie/apps/events/apps.py +2 -2
- irie/apps/events/models.py +1 -1
- irie/apps/events/views.py +2 -2
- irie/apps/events/views_events.py +3 -3
- irie/apps/inventory/apps.py +1 -1
- irie/apps/inventory/forms.py +1 -1
- irie/apps/inventory/models.py +2 -2
- irie/apps/inventory/urls.py +1 -1
- irie/apps/inventory/views.py +11 -11
- irie/apps/networks/apps.py +1 -1
- irie/apps/networks/networks.py +1 -1
- irie/apps/networks/urls.py +3 -3
- irie/apps/prediction/apps.py +1 -1
- irie/apps/prediction/forms.py +1 -1
- irie/apps/prediction/models.py +1 -1
- irie/apps/prediction/views.py +4 -4
- irie/apps/prediction/views_api.py +3 -3
- irie/apps/site/config.py +2 -2
- irie/apps/site/urls.py +1 -1
- irie/apps/site/view_sdof.py +1 -1
- irie/core/settings.py +9 -9
- irie/core/urls.py +8 -8
- irie/init/__main__.py +27 -0
- irie/init/bridges.py +1098 -0
- irie/init/calid.py +34 -0
- irie/init/getCGSData.py +104 -0
- irie/init/getCGSevents.py +16 -0
- irie/init/getNBIData.py +201 -0
- irie/init/init_assets.py +172 -0
- irie/init/init_corridors.py +39 -0
- irie/init/init_predictors.py +46 -0
- irie/mgmt.py +24 -0
- {irie-0.0.0.dist-info → irie-0.0.1.dist-info}/METADATA +24 -5
- irie-0.0.1.dist-info/RECORD +109 -0
- irie/apps/authentication/migrations/__init__.py +0 -1
- irie/apps/evaluation/migrations/0001_initial.py +0 -25
- irie/apps/evaluation/migrations/0002_remove_evaluation_cesmd.py +0 -17
- irie/apps/evaluation/migrations/0003_evaluation_asset.py +0 -20
- irie/apps/evaluation/migrations/__init__.py +0 -0
- irie/apps/events/migrations/0001_initial.py +0 -27
- irie/apps/events/migrations/0002_alter_event_id.py +0 -18
- irie/apps/events/migrations/0003_event_cesmd.py +0 -19
- irie/apps/events/migrations/0004_event_record_identifier.py +0 -19
- irie/apps/events/migrations/0005_event_asset.py +0 -21
- irie/apps/events/migrations/0006_alter_event_event_file.py +0 -18
- irie/apps/events/migrations/__init__.py +0 -0
- irie/apps/inventory/migrations/0001_initial.py +0 -31
- irie/apps/inventory/migrations/0002_assetevaluationmodel_cesmd.py +0 -19
- irie/apps/inventory/migrations/0003_auto_20230520_2030.py +0 -23
- irie/apps/inventory/migrations/0004_asset.py +0 -27
- irie/apps/inventory/migrations/0005_auto_20230731_1802.py +0 -23
- irie/apps/inventory/migrations/0006_auto_20230731_1816.py +0 -28
- irie/apps/inventory/migrations/0007_auto_20230731_1827.py +0 -24
- irie/apps/inventory/migrations/0008_asset_is_complete.py +0 -19
- irie/apps/inventory/migrations/0009_auto_20230731_1842.py +0 -29
- irie/apps/inventory/migrations/0010_auto_20230801_0025.py +0 -23
- irie/apps/inventory/migrations/0011_alter_asset_cgs_data.py +0 -18
- irie/apps/inventory/migrations/0012_corridor.py +0 -22
- irie/apps/inventory/migrations/0013_alter_asset_cesmd.py +0 -18
- irie/apps/inventory/migrations/0014_alter_asset_cesmd.py +0 -18
- irie/apps/inventory/migrations/__init__.py +0 -0
- irie/apps/networks/migrations/0001_initial.py +0 -26
- irie/apps/networks/migrations/__init__.py +0 -0
- irie/apps/prediction/migrations/0001_initial.py +0 -32
- irie/apps/prediction/migrations/0002_auto_20230731_1801.py +0 -27
- irie/apps/prediction/migrations/0003_rename_assetevaluationmodel_evaluation.py +0 -18
- irie/apps/prediction/migrations/0004_delete_evaluation.py +0 -16
- irie/apps/prediction/migrations/0005_predictormodel_protocol.py +0 -18
- irie/apps/prediction/migrations/0006_alter_predictormodel_protocol.py +0 -18
- irie/apps/prediction/migrations/0007_predictormodel_active.py +0 -19
- irie/apps/prediction/migrations/0008_predictormodel_description.py +0 -18
- irie/apps/prediction/migrations/0009_predictormodel_entry_point.py +0 -19
- irie/apps/prediction/migrations/0010_alter_predictormodel_entry_point.py +0 -18
- irie/apps/prediction/migrations/0011_remove_predictormodel_entry_point.py +0 -17
- irie/apps/prediction/migrations/0012_predictormodel_entry_point.py +0 -18
- irie/apps/prediction/migrations/0013_predictormodel_metrics.py +0 -18
- irie/apps/prediction/migrations/0014_auto_20240930_0004.py +0 -28
- irie/apps/prediction/migrations/0015_alter_predictormodel_render_file.py +0 -18
- irie/apps/prediction/migrations/__init__.py +0 -0
- irie/apps/site/migrations/__init__.py +0 -1
- irie-0.0.0.dist-info/RECORD +0 -145
- /irie/{apps/documents/migrations → init}/__init__.py +0 -0
- {irie-0.0.0.dist-info → irie-0.0.1.dist-info}/WHEEL +0 -0
- {irie-0.0.0.dist-info → irie-0.0.1.dist-info}/entry_points.txt +0 -0
- {irie-0.0.0.dist-info → irie-0.0.1.dist-info}/top_level.txt +0 -0
irie/init/calid.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
|
|
2
|
+
CESMD = {
|
|
3
|
+
# "CE89324": {},
|
|
4
|
+
# "CE58700": "CE58700", # CE58700
|
|
5
|
+
"04-0236": "CE89324", # painter
|
|
6
|
+
"04-0228": "CE89686", # eureka-samoa
|
|
7
|
+
"10-0299": "CE79421", # CE79421
|
|
8
|
+
"23-0015R": "CE68184", # carquinez-east-northbound
|
|
9
|
+
"28-0352L": "CE68185", # carquinez-west-southbound
|
|
10
|
+
"28-0153": "CE68682", # CE68682
|
|
11
|
+
"33-0214L": "CE58658", # hayward
|
|
12
|
+
"34-0003": "CE58632", # CE58632
|
|
13
|
+
"34-0006": "CE58600", # CE58600
|
|
14
|
+
"34-0006": "CE58601", # CE58601
|
|
15
|
+
"04-0016R": "CE89973", # CE89973
|
|
16
|
+
"04-0170": "CE89708", # CE89708
|
|
17
|
+
"04-0229": "CE89735", # CE89735
|
|
18
|
+
"04-0230": "CE89736", # CE89736
|
|
19
|
+
|
|
20
|
+
"43-0031E": "CE47315", # CE47315
|
|
21
|
+
"47-0048": "CE54730", # crowley
|
|
22
|
+
"50-0271": "CE24775", # CE24775
|
|
23
|
+
"50-0340": "CE33742", # ridgecrest
|
|
24
|
+
"53-1471": "CE14406", # CE14406
|
|
25
|
+
"53-1794": "CE24706", # CE24706
|
|
26
|
+
"53-2791": "CE24704", # CE24704
|
|
27
|
+
"53-2795F": "CE24694", # sylmar
|
|
28
|
+
"54-0823G": "CE23631", # bernardino
|
|
29
|
+
"55-0225": "CE13795", # capistrano
|
|
30
|
+
"56-0586G": "CE13705", # CE13705
|
|
31
|
+
"58-0215": "CE01336", # meloland
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
CALID = {v: k for k, v in CESMD.items()}
|
irie/init/getCGSData.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# ____ _____ _____ ______ ___
|
|
3
|
+
# | _ \| __ \ /\ / ____| ____|__ \
|
|
4
|
+
# | |_) | |__) | / \ | | | |__ ) |
|
|
5
|
+
# | _ <| _ / / /\ \| | | __| / /
|
|
6
|
+
# | |_) | | \ \/ ____ \ |____| |____ / /_
|
|
7
|
+
# |____/|_| \_\/ \_\_____|______|____|
|
|
8
|
+
#
|
|
9
|
+
# claudio perez
|
|
10
|
+
"""
|
|
11
|
+
This script takes an HTML page, extracts its tables, and prints
|
|
12
|
+
a JSON representation of the data.
|
|
13
|
+
|
|
14
|
+
Example:
|
|
15
|
+
$ pandoc ../el-centro/documentation/summary-table.md 2>/dev/null | python html2json.py
|
|
16
|
+
"""
|
|
17
|
+
import sys
|
|
18
|
+
import requests
|
|
19
|
+
from bs4 import BeautifulSoup # install package 'beautifulsoup4'
|
|
20
|
+
|
|
21
|
+
# from elstir.contrib.pandoc import Pandoc
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def pull_csmip(csmip_id:str)->str:
|
|
25
|
+
return requests.get(
|
|
26
|
+
f"https://www.strongmotioncenter.org/cgi-bin/CESMD/stationhtml.pl?stationID={csmip_id}&network=CGS"
|
|
27
|
+
).text
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def dict2latex(data:dict, i=0)->str:
|
|
31
|
+
"""
|
|
32
|
+
take a dictionary and return a string defining a LaTeX table.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
head = f"""
|
|
36
|
+
\\begin{{table}}[!htbp]
|
|
37
|
+
\\centering
|
|
38
|
+
\\begin{{tabular}}{{|p{{35mm}}|p{{8cm}}|}} \\hline
|
|
39
|
+
"""
|
|
40
|
+
body = "\n ".join([f"{k} & {v}\\\\ \\hline" for k,v in data.items()])
|
|
41
|
+
tail = f"""
|
|
42
|
+
\\end{{tabular}}
|
|
43
|
+
\\label{{tab:num-{i}}}
|
|
44
|
+
\\end{{table}}
|
|
45
|
+
\\vspace*{{2cm}}
|
|
46
|
+
"""
|
|
47
|
+
return head + " " + body + tail
|
|
48
|
+
|
|
49
|
+
def table2dict(html_table, output=None)->dict:
|
|
50
|
+
"""
|
|
51
|
+
This function takes HTML tables with rows of the form:
|
|
52
|
+
|
|
53
|
+
<tr>
|
|
54
|
+
<td>key1</td> <td>value1</td>
|
|
55
|
+
</tr>
|
|
56
|
+
"""
|
|
57
|
+
output = {} if output is None else output
|
|
58
|
+
tree = html_table
|
|
59
|
+
tables = tree.find_all("table")
|
|
60
|
+
|
|
61
|
+
if tables:
|
|
62
|
+
# the tree has multiple tables, recurse
|
|
63
|
+
table_dicts = [table2dict(table) for table in tables if table]
|
|
64
|
+
return [table for table in table_dicts if table]
|
|
65
|
+
|
|
66
|
+
else:
|
|
67
|
+
table = tree
|
|
68
|
+
|
|
69
|
+
for row in table.find_all("tr"):
|
|
70
|
+
header = row.find("th")
|
|
71
|
+
if header:
|
|
72
|
+
output.update({"headers": header.text.split()})
|
|
73
|
+
|
|
74
|
+
data = row.find_all("td")
|
|
75
|
+
if "Photograph" in data[0].text:
|
|
76
|
+
continue
|
|
77
|
+
elif data[0].text == "" and len(data) > 1:
|
|
78
|
+
print(data[1].find("img").attrs["src"], file=sys.stderr)
|
|
79
|
+
continue
|
|
80
|
+
elif len(data) > 1 and data[1].text == "\u00a0":
|
|
81
|
+
continue
|
|
82
|
+
|
|
83
|
+
try:
|
|
84
|
+
output.update({data[0].text: data[1].text})
|
|
85
|
+
except:
|
|
86
|
+
pass
|
|
87
|
+
|
|
88
|
+
return output
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
if __name__=="__main__":
|
|
92
|
+
import sys
|
|
93
|
+
import json
|
|
94
|
+
# from init.bridges import BRIDGES
|
|
95
|
+
exec(open("init/bridges.py").read())
|
|
96
|
+
|
|
97
|
+
dat = {}
|
|
98
|
+
for cesmd in BRIDGES:
|
|
99
|
+
html = pull_csmip(cesmd)
|
|
100
|
+
dat.update({cesmd: table2dict(BeautifulSoup(html, "html.parser"))})
|
|
101
|
+
|
|
102
|
+
print(json.dumps(dat, indent=4))
|
|
103
|
+
|
|
104
|
+
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
import sys
|
|
3
|
+
import cesmd.search
|
|
4
|
+
|
|
5
|
+
# scrape.py <cesmd> <out-dir>
|
|
6
|
+
outdir = sys.argv[1]
|
|
7
|
+
|
|
8
|
+
cesmd.search.get_records(outdir,
|
|
9
|
+
"cchern@berkeley.edu",
|
|
10
|
+
station_code=sys.argv[1],
|
|
11
|
+
#unpack=True,
|
|
12
|
+
network="CE",
|
|
13
|
+
include_inactive=True,
|
|
14
|
+
process_level="processed",
|
|
15
|
+
)
|
|
16
|
+
|
irie/init/getNBIData.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
#===----------------------------------------------------------------------===#
|
|
2
|
+
#
|
|
3
|
+
# STAIRLab -- STructural Artificial Intelligence Laboratory
|
|
4
|
+
#
|
|
5
|
+
#===----------------------------------------------------------------------===#
|
|
6
|
+
#
|
|
7
|
+
# First run without arguments to generate list of structure numbers and
|
|
8
|
+
# save to JSON. Then run with that JSON as argv[1] to pull inventory data.
|
|
9
|
+
#
|
|
10
|
+
# Adapted from:
|
|
11
|
+
# https://github.com/psychogeekir/ScrapeNBIBridgeInfo/raw/master/getNBIClimateData.py
|
|
12
|
+
#
|
|
13
|
+
# Claudio M. Perez
|
|
14
|
+
#
|
|
15
|
+
import sys
|
|
16
|
+
import json
|
|
17
|
+
import requests
|
|
18
|
+
from tqdm import tqdm
|
|
19
|
+
|
|
20
|
+
def getBridgeList(headers, start_page=1, totalpages=3, pagesize=10, totalbridges=24, page_nums=None, **kwds):
|
|
21
|
+
url = 'https://infobridge.fhwa.dot.gov/Data/GetAllBridges'
|
|
22
|
+
|
|
23
|
+
payload = {
|
|
24
|
+
"isShowBridgesApplied":True,
|
|
25
|
+
"gridParam": {
|
|
26
|
+
"isShowBridgesApplied":True, "IsFilterApplied":False, "SelectedFilters":None,
|
|
27
|
+
"SortOrder":"asc", "SortIndex": "STATE_CODE",
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
pages = []
|
|
32
|
+
def filter(row):
|
|
33
|
+
return int(row["STATE_CODE"]) > 0
|
|
34
|
+
|
|
35
|
+
if page_nums is None:
|
|
36
|
+
page_nums = range(start_page, totalpages + 1)
|
|
37
|
+
|
|
38
|
+
for pageno in page_nums:
|
|
39
|
+
try:
|
|
40
|
+
payload["gridParam"]["PageNumber"] = pageno
|
|
41
|
+
payload["gridParam"]["PageSize"] = pagesize
|
|
42
|
+
|
|
43
|
+
r = requests.post(url, headers=headers, data=json.dumps(payload))
|
|
44
|
+
|
|
45
|
+
if r.status_code == 200:
|
|
46
|
+
try:
|
|
47
|
+
resp = json.loads(eval(r.content.decode('utf-8'))) # [1:-1].replace("\\", ""))
|
|
48
|
+
except:
|
|
49
|
+
print(f"Failed to get page {pageno}", file=sys.stderr)
|
|
50
|
+
continue
|
|
51
|
+
|
|
52
|
+
bridges = [
|
|
53
|
+
{'BRIDGE_YEARLY_ID': row['BRIDGE_YEARLY_ID'],
|
|
54
|
+
'STRUCTURE_NUMBER': row['STRUCTURE_NUMBER'].strip()}
|
|
55
|
+
for row in resp["Results"]["rows"] # if filter(row)
|
|
56
|
+
]
|
|
57
|
+
pages.extend(bridges)
|
|
58
|
+
print(pageno, len(pages), len(bridges), resp["Results"]["rows"][-1]["STATE_NAME"], file=sys.stderr)
|
|
59
|
+
|
|
60
|
+
except KeyboardInterrupt:
|
|
61
|
+
break
|
|
62
|
+
|
|
63
|
+
return pages
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def getNBIData(headers, bridgeTable, years,
|
|
67
|
+
keep_query=False):
|
|
68
|
+
|
|
69
|
+
url = 'https://infobridge.fhwa.dot.gov/Data/getBridgeInformation'
|
|
70
|
+
|
|
71
|
+
_headers = headers.copy()
|
|
72
|
+
# payload = {
|
|
73
|
+
# "requestModel": {
|
|
74
|
+
# "SELECTED_TAB": "OverviewTab",
|
|
75
|
+
# "SELECTED_YEAR_ID": None,
|
|
76
|
+
# "IS_NEW_RECORD": True,
|
|
77
|
+
# "IS_YEAR_SELECTED": False,
|
|
78
|
+
# "Is_Overview_Bridge_Selected": False,
|
|
79
|
+
# "SELECTED_YEAR": None,
|
|
80
|
+
# "CURRENT_YEARLY_ID": "25099893",
|
|
81
|
+
# "IS_NBI_TREE_SELECTED": False,
|
|
82
|
+
# "Folder_Name": None,
|
|
83
|
+
# "tabChange": False,
|
|
84
|
+
# "BRIDGE_YEARLY_ID": "25099893",
|
|
85
|
+
# "NEW_BRIDGE_ID": 58813,
|
|
86
|
+
# "SELECTED_NDE_TAB": "General"
|
|
87
|
+
# }
|
|
88
|
+
# }
|
|
89
|
+
|
|
90
|
+
payload = {
|
|
91
|
+
"requestModel":{
|
|
92
|
+
"SELECTED_TAB": "NBITab",
|
|
93
|
+
"SELECTED_YEAR_ID": None,
|
|
94
|
+
"IS_NEW_RECORD": False,
|
|
95
|
+
"IS_YEAR_SELECTED": False,
|
|
96
|
+
"Is_Overview_Bridge_Selected": False,
|
|
97
|
+
"NEW_BRIDGE_ID": 0,
|
|
98
|
+
"STRUCTURE_NUMBER": None,
|
|
99
|
+
"STATE_NAME": None,
|
|
100
|
+
"STATE_CODE": 0,
|
|
101
|
+
"IS_EXPERIMENTAL": False,
|
|
102
|
+
"SELECTED_NDE_TAB": "General",
|
|
103
|
+
#"MERRA_ID": 0,"IS_NBI_TREE_SELECTED": False,"Folder_Name": None,"tabChange": False,
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
referer = 'https://infobridge.fhwa.dot.gov/Data/BridgeDetail/'
|
|
109
|
+
data = {}
|
|
110
|
+
|
|
111
|
+
for i,bridge in enumerate(tqdm(bridgeTable)):
|
|
112
|
+
|
|
113
|
+
BRIDGE_YEARLY_ID = bridge['BRIDGE_YEARLY_ID']
|
|
114
|
+
STRUCTURE_NUMBER = bridge['STRUCTURE_NUMBER']
|
|
115
|
+
|
|
116
|
+
data[STRUCTURE_NUMBER] = []
|
|
117
|
+
|
|
118
|
+
for year in years:
|
|
119
|
+
_headers['referer'] = referer + str(BRIDGE_YEARLY_ID)
|
|
120
|
+
|
|
121
|
+
payload["requestModel"].update({
|
|
122
|
+
"SELECTED_YEAR": year,
|
|
123
|
+
"CURRENT_YEARLY_ID": BRIDGE_YEARLY_ID,
|
|
124
|
+
"BRIDGE_YEARLY_ID": BRIDGE_YEARLY_ID,
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
r = requests.post(url, data=json.dumps(payload), headers=_headers)
|
|
128
|
+
|
|
129
|
+
if r.status_code == 200:
|
|
130
|
+
htmlcontent = r.content.decode('utf-8')
|
|
131
|
+
try:
|
|
132
|
+
data[STRUCTURE_NUMBER][0] = {
|
|
133
|
+
k: (
|
|
134
|
+
v if k != "Results" else {
|
|
135
|
+
kk: vv for kk, vv in v.items() if (kk != "NBIDataQuery" or keep_query)
|
|
136
|
+
}
|
|
137
|
+
) for k, v in json.loads(htmlcontent).items()
|
|
138
|
+
}
|
|
139
|
+
except Exception as e:
|
|
140
|
+
print(f">> Error: {e}", file=sys.stderr)
|
|
141
|
+
continue
|
|
142
|
+
|
|
143
|
+
else:
|
|
144
|
+
print(f">> Error ({year}) {r.status_code}: {r.content}", file=sys.stderr)
|
|
145
|
+
|
|
146
|
+
if i % 500 == 0:
|
|
147
|
+
with open(f"nbi_data-{i}.json", "w") as f:
|
|
148
|
+
json.dump(data, f, indent=2)
|
|
149
|
+
|
|
150
|
+
return data
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
if __name__ == '__main__':
|
|
154
|
+
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0"
|
|
155
|
+
request_verification_token = 'CfDJ8M6CuWz5hhxGnmUVXw2yDHQlfeNDzVoF03IbAJ0p3LdaDW7poklPvy74ykYda-qwcrtUXD4rnNzn583Ug7PWbR9IlomGzQh1OQIw_pa9d5TNwdN5p77SDfIfz3yq1nWPzxemEn_8bbh7TGGK9FIwcRY'
|
|
156
|
+
cookie = "_ga=GA1.1.478241025.1718907711; _ga_0623JYSC1Q=GS1.1.1718922743.2.0.1718922743.0.0.0; _ga_VW1SFWJKBB=GS1.1.1730789269.3.0.1730789272.0.0.0; _ga_CSLL4ZEK4L=GS1.1.1730789269.3.0.1730789272.0.0.0; _ga_NQ5ZN114SB=GS1.1.1730789269.3.0.1730789272.0.0.0; .AspNetCore.Session=CfDJ8M6CuWz5hhxGnmUVXw2yDHRQxNlIdqc8pBGKOJhMcHphMelhCyOQD7cnzYLVUWcsfCE8KOO8TNogarX5FbmvNQeSW1pTphWgR%2B6RLzPiUWuR4yPiDmb6rg82isfHqoEBhFoziXpFlU2o9pMgQICLsy7WbaeZbSgOl6FTg5Y0vLQ5; __RequestVerificationToken=CfDJ8M6CuWz5hhxGnmUVXw2yDHQXNjHWpjZ61I-CMSrl0yWsdWpCyt2QhUoeZ2L2aY0sqNpGy-wrD8ToMph6-wbfcRPpqORdlVci0ghxWu-3i4PCuWsiOkq90E1WupEYErSXnhsQVwHHGcD63WI7qyXZd7w; _ga_GNYE9X3V7H=GS1.1.1730825963.2.1.1730825988.0.0.0"
|
|
157
|
+
|
|
158
|
+
headers = {
|
|
159
|
+
'authority': 'infobridge.fhwa.dot.gov',
|
|
160
|
+
'origin': 'https://infobridge.fhwa.dot.gov',
|
|
161
|
+
'sec-fetch-site': 'same-origin',
|
|
162
|
+
'sec-fetch-mode': 'cors',
|
|
163
|
+
'accept-encoding': 'gzip, deflate, br',
|
|
164
|
+
'accept-language': 'en-US,en;q=0.9',
|
|
165
|
+
|
|
166
|
+
'__requestverificationtoken': request_verification_token,
|
|
167
|
+
'user-agent': user_agent,
|
|
168
|
+
'cookie': cookie
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
if len(sys.argv) == 1:
|
|
172
|
+
|
|
173
|
+
headers.update({
|
|
174
|
+
'x-requested-with': 'XMLHttpRequest',
|
|
175
|
+
'content-type': 'application/json; charset=UTF-8',
|
|
176
|
+
'accept': 'application/json, text/javascript, */*; q=0.01',
|
|
177
|
+
'referer': 'https://infobridge.fhwa.dot.gov/Data',
|
|
178
|
+
})
|
|
179
|
+
|
|
180
|
+
bridgeTable = getBridgeList(headers, start_page=511, totalpages=808, pagesize=100)
|
|
181
|
+
print(json.dumps(bridgeTable, indent=2))
|
|
182
|
+
sys.exit()
|
|
183
|
+
|
|
184
|
+
else:
|
|
185
|
+
headers.update({
|
|
186
|
+
'datatype': 'json',
|
|
187
|
+
'content-type': 'application/json; charset=UTF-8',
|
|
188
|
+
'accept': 'application/json, text/plain, */*'
|
|
189
|
+
})
|
|
190
|
+
bridgeTable = json.load(open(sys.argv[1]))
|
|
191
|
+
bridgeTable = [
|
|
192
|
+
i for i in bridgeTable
|
|
193
|
+
if " " in i["STRUCTURE_NUMBER"] and len(i["STRUCTURE_NUMBER"]) in {7, 8}
|
|
194
|
+
]
|
|
195
|
+
|
|
196
|
+
# calids = list(map(str.strip, open("init/calid.txt").readlines()))
|
|
197
|
+
# bridgeTable = [i for i in bridgeTable if i["STRUCTURE_NUMBER"] in calids]
|
|
198
|
+
|
|
199
|
+
nbi_data = getNBIData(headers, bridgeTable[:], years=(2024,)) #range(2020, 2024))
|
|
200
|
+
print(json.dumps(nbi_data, indent=2))
|
|
201
|
+
|
irie/init/init_assets.py
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""
|
|
2
|
+
python manage.py shell < scripts/make_assets.py
|
|
3
|
+
"""
|
|
4
|
+
try:
|
|
5
|
+
import orjson as json
|
|
6
|
+
except ImportError:
|
|
7
|
+
import json
|
|
8
|
+
|
|
9
|
+
from collections import defaultdict
|
|
10
|
+
from irie.apps.inventory.models import Asset, Corridor
|
|
11
|
+
from init.calid import CALID, CESMD
|
|
12
|
+
from init.bridges import BRIDGES
|
|
13
|
+
|
|
14
|
+
DRY = False
|
|
15
|
+
UPDATE_ASSETS = True
|
|
16
|
+
|
|
17
|
+
DISTRICTS = {
|
|
18
|
+
# "01 - District 1",
|
|
19
|
+
"04 - District 4",
|
|
20
|
+
# "05 - District 5",
|
|
21
|
+
# "06 - District 6",
|
|
22
|
+
# "07 - District 7",
|
|
23
|
+
# "08 - District 8",
|
|
24
|
+
# "09 - District 9",
|
|
25
|
+
# "11 - District 11",
|
|
26
|
+
# "12 - District 12"
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
MIN_ADTT = 1_500
|
|
30
|
+
MIN_ADT = 15_000
|
|
31
|
+
|
|
32
|
+
SKIP_DESIGN = {
|
|
33
|
+
"19 - Culvert"
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
#-----------------------------------
|
|
37
|
+
|
|
38
|
+
NBI_FILE = "data/nbi/04.json" #"nbi_data-500.json" # "data/nbi_data-california.json" # "data/nbi-california-2024.json" #
|
|
39
|
+
with open(NBI_FILE) as f:
|
|
40
|
+
NBI_DATA = json.loads(f.read())
|
|
41
|
+
|
|
42
|
+
with open("data/cgs_data.json") as f:
|
|
43
|
+
CGS_DATA = json.loads(f.read())
|
|
44
|
+
|
|
45
|
+
# NBI_DATA = json.load(open("data/nbi_data-california.json"))
|
|
46
|
+
# NBI_DATA = json.load(open("data/nbi_data-california.json"))
|
|
47
|
+
|
|
48
|
+
def find_bridge(bridges, calid):
|
|
49
|
+
for bridge in bridges.values():
|
|
50
|
+
if bridge["calid"].split(" ")[0] == calid:
|
|
51
|
+
return bridge
|
|
52
|
+
return {}
|
|
53
|
+
|
|
54
|
+
def get_nbi(calid, missing_ok=False):
|
|
55
|
+
data = defaultdict(dict)
|
|
56
|
+
|
|
57
|
+
if missing_ok and calid not in NBI_DATA:
|
|
58
|
+
return None
|
|
59
|
+
|
|
60
|
+
blocks = NBI_DATA[calid]
|
|
61
|
+
for row in blocks[-1]["Results"]["NBIData"]["NBIDataList"]:
|
|
62
|
+
data[row["TABLE_NAME"]][row["EXPANDED_FIELD_ALIAS"]] = row["FIELD_VALUE"]
|
|
63
|
+
|
|
64
|
+
return dict(data)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def get_route(bridge):
|
|
68
|
+
return "-".join(bridge["NBI_BRIDGE"]["Location"].split("-")[:3])
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def skip(bridge, routes):
|
|
72
|
+
return not (
|
|
73
|
+
(
|
|
74
|
+
get_route(bridge) in routes
|
|
75
|
+
and bridge["NBI_BRIDGE"]["Highway Agency District"] in DISTRICTS
|
|
76
|
+
and bridge["NBI_POSTING_STATUS"]["Structure Operational Status Code"] == "A - Open"
|
|
77
|
+
) or (
|
|
78
|
+
bridge["NBI_BRIDGE"]["Type of Service on Bridge Code"] == "1 - Highway"
|
|
79
|
+
and bridge["NBI_BRIDGE"]["Owner Agency"] == "1 - State Highway Agency"
|
|
80
|
+
and bridge["NBI_SUPERSTRUCTURE_DECK"]["Main Span Design"] not in SKIP_DESIGN
|
|
81
|
+
and (
|
|
82
|
+
"Concrete" in bridge["NBI_SUPERSTRUCTURE_DECK"]["Main Span Material"]
|
|
83
|
+
or "Steel" in bridge["NBI_SUPERSTRUCTURE_DECK"]["Main Span Material"]
|
|
84
|
+
)
|
|
85
|
+
# and bridge["NBI_FEATURE"]["Inventory Route NHS Code"] == "1 - On NHS"
|
|
86
|
+
and int(bridge["NBI_FEATURE"]["Average Daily Truck Traffic (Volume)"]) >= MIN_ADTT
|
|
87
|
+
and int(bridge["NBI_FEATURE"]["Average Daily Traffic"]) >= MIN_ADT
|
|
88
|
+
# and bridge["NBI_BRIDGE"]["Coulverts Condition Rating"] == "N - Not a culvert"
|
|
89
|
+
)
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
# 1. Collect routes of interest
|
|
94
|
+
ROUTES = set()
|
|
95
|
+
for bridge in BRIDGES.values():
|
|
96
|
+
nbi = get_nbi(bridge["calid"].split(" ")[0].replace("-", " "), missing_ok=True)
|
|
97
|
+
if nbi is not None:
|
|
98
|
+
ROUTES.add(get_route(nbi))
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
count = 0
|
|
102
|
+
|
|
103
|
+
CORRIDORS = defaultdict(set)
|
|
104
|
+
|
|
105
|
+
for item in NBI_DATA:
|
|
106
|
+
calid = item.replace(" ", "-")
|
|
107
|
+
nbi = get_nbi(calid.replace("-", " "))
|
|
108
|
+
config = find_bridge(BRIDGES, calid)
|
|
109
|
+
cesmd = CESMD.get(calid, None)
|
|
110
|
+
try:
|
|
111
|
+
if skip(nbi, ROUTES):
|
|
112
|
+
continue
|
|
113
|
+
except:
|
|
114
|
+
print("Failed to skip ", calid)
|
|
115
|
+
continue
|
|
116
|
+
|
|
117
|
+
count += 1
|
|
118
|
+
print(calid, f"({cesmd = })")
|
|
119
|
+
|
|
120
|
+
cname = get_route(nbi)
|
|
121
|
+
|
|
122
|
+
CORRIDORS[cname].add(calid)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
if DRY:
|
|
126
|
+
continue
|
|
127
|
+
|
|
128
|
+
try:
|
|
129
|
+
asset = Asset.objects.get(calid=calid)
|
|
130
|
+
if UPDATE_ASSETS:
|
|
131
|
+
if cesmd is not None:
|
|
132
|
+
asset.cesmd = cesmd
|
|
133
|
+
asset.cgs_data = CGS_DATA.get(cesmd, {})
|
|
134
|
+
asset.nbi_data = nbi
|
|
135
|
+
asset.save()
|
|
136
|
+
|
|
137
|
+
print(">> Saved ", calid, f"({cesmd = })")
|
|
138
|
+
|
|
139
|
+
except:
|
|
140
|
+
if nbi is None:
|
|
141
|
+
print(">> Skipping ", calid)
|
|
142
|
+
continue
|
|
143
|
+
|
|
144
|
+
name = config.get("name", nbi["NBI_BRIDGE"]["Location"])
|
|
145
|
+
asset = Asset(cesmd=cesmd,
|
|
146
|
+
calid=calid,
|
|
147
|
+
name = name,
|
|
148
|
+
cgs_data = CGS_DATA.get(cesmd, {}),
|
|
149
|
+
nbi_data = nbi,
|
|
150
|
+
is_complete=False)
|
|
151
|
+
asset.save()
|
|
152
|
+
print(asset)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
try:
|
|
156
|
+
corridor = Corridor.objects.get(name=cname)
|
|
157
|
+
corridor.save()
|
|
158
|
+
print(">> Saved ", cname)
|
|
159
|
+
continue
|
|
160
|
+
|
|
161
|
+
except:
|
|
162
|
+
corridor = Corridor(name=cname)
|
|
163
|
+
corridor.save()
|
|
164
|
+
print(corridor)
|
|
165
|
+
|
|
166
|
+
corridor.assets.add(asset)
|
|
167
|
+
|
|
168
|
+
del nbi
|
|
169
|
+
|
|
170
|
+
print(f"Created {count} of {len(NBI_DATA)} assets")
|
|
171
|
+
|
|
172
|
+
print(f"Created {len(CORRIDORS)} corridors")
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""
|
|
2
|
+
To run:
|
|
3
|
+
python manage.py shell < scripts/init_corridors.py
|
|
4
|
+
|
|
5
|
+
This script depends on the files:
|
|
6
|
+
soga_corridors.json
|
|
7
|
+
|
|
8
|
+
This file is created using the script make_corridors.py
|
|
9
|
+
which takes in corridor_line.geojson and soga_corridors.csv
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import sys
|
|
13
|
+
import csv
|
|
14
|
+
import json
|
|
15
|
+
from collections import defaultdict
|
|
16
|
+
from irie.apps.inventory.models import Asset, Corridor
|
|
17
|
+
|
|
18
|
+
if True:
|
|
19
|
+
with open("data/soga_corridors.json") as f:
|
|
20
|
+
corridors = json.load(f)
|
|
21
|
+
|
|
22
|
+
for cdata in corridors:
|
|
23
|
+
cname = cdata["name"]
|
|
24
|
+
|
|
25
|
+
try:
|
|
26
|
+
corridor = Corridor.objects.get(name=cname)
|
|
27
|
+
|
|
28
|
+
except:
|
|
29
|
+
corridor = Corridor(name=cname)
|
|
30
|
+
|
|
31
|
+
for calid in cdata["bridges"]:
|
|
32
|
+
# corridor.assets.add(Asset.objects.get(calid=calid))
|
|
33
|
+
try:
|
|
34
|
+
corridor.assets.add(Asset.objects.get(calid=calid))
|
|
35
|
+
print(f"Added {calid} to {corridor.name}")
|
|
36
|
+
except Exception as e:
|
|
37
|
+
print(f"Failed to find assed with calid {calid} ({e})")
|
|
38
|
+
|
|
39
|
+
corridor.save()
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Assets must already be created
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
from irie.apps.inventory.models import Asset
|
|
8
|
+
from irie.apps.prediction.models import PredictorModel
|
|
9
|
+
from init.bridges import BRIDGES
|
|
10
|
+
|
|
11
|
+
for bridge in BRIDGES.values():
|
|
12
|
+
print(bridge["cesmd"])
|
|
13
|
+
for conf in bridge.get("predictors", []):
|
|
14
|
+
print(">> ", conf["name"])
|
|
15
|
+
try:
|
|
16
|
+
"a" + 1
|
|
17
|
+
pred = PredictorModel.objects.get(cesmd=bridge["cesmd"])
|
|
18
|
+
pred.config = conf["config"]
|
|
19
|
+
pred.name = conf["name"]
|
|
20
|
+
pred.save()
|
|
21
|
+
|
|
22
|
+
print(">> Saved ", bridge["cesmd"])
|
|
23
|
+
continue
|
|
24
|
+
except:
|
|
25
|
+
protocol = PredictorModel.Protocol.TYPE2
|
|
26
|
+
for type in PredictorModel.Protocol:
|
|
27
|
+
print(f" {type._name_}: {conf['protocol']}")
|
|
28
|
+
if str(type) == conf["protocol"]:
|
|
29
|
+
protocol = type
|
|
30
|
+
|
|
31
|
+
# if protocol is None:
|
|
32
|
+
# raise ValueError(f"Unknown predictor protocol: {conf['protocol']}")
|
|
33
|
+
config = conf.get("config", {})
|
|
34
|
+
|
|
35
|
+
a = PredictorModel(asset = Asset.objects.get(cesmd=bridge["cesmd"]),
|
|
36
|
+
name = conf["name"],
|
|
37
|
+
entry_point = conf["entry_point"],
|
|
38
|
+
config = config,
|
|
39
|
+
description = conf.get("description", ""),
|
|
40
|
+
active = True,
|
|
41
|
+
metrics = list(conf.get("metrics", [])),
|
|
42
|
+
protocol = protocol
|
|
43
|
+
)
|
|
44
|
+
a.save()
|
|
45
|
+
print(a)
|
|
46
|
+
|
irie/mgmt.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
#===----------------------------------------------------------------------===#
|
|
3
|
+
#
|
|
4
|
+
# STAIRLab -- STructural Artificial Intelligence Laboratory
|
|
5
|
+
#
|
|
6
|
+
#===----------------------------------------------------------------------===#
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
def main():
|
|
12
|
+
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'irie.core.settings')
|
|
13
|
+
try:
|
|
14
|
+
from django.core.management import execute_from_command_line
|
|
15
|
+
except ImportError as exc:
|
|
16
|
+
raise ImportError(
|
|
17
|
+
"Couldn't import Django. Are you sure it's installed and "
|
|
18
|
+
"available on your PYTHONPATH environment variable? Did you "
|
|
19
|
+
"forget to activate a virtual environment?"
|
|
20
|
+
) from exc
|
|
21
|
+
execute_from_command_line(sys.argv)
|
|
22
|
+
|
|
23
|
+
if __name__ == '__main__':
|
|
24
|
+
main()
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: irie
|
|
3
|
-
Version: 0.0.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Infrastructure Resilience Engine
|
|
5
5
|
Author-email: wd <5018-006+wd@users.noreply.github.com>, ab <52-93467+ab@users.noreply.github.com>
|
|
6
6
|
Project-URL: Repository, https://github.com/STAIRLab/irie
|
|
7
7
|
Project-URL: Documentation, https://structures.live
|
|
@@ -31,11 +31,30 @@ Requires-Dist: opensees
|
|
|
31
31
|
Requires-Dist: quakeio
|
|
32
32
|
Requires-Dist: bottle
|
|
33
33
|
Requires-Dist: openbim
|
|
34
|
+
Requires-Dist: Django==5.1.2
|
|
35
|
+
Requires-Dist: asgiref==3.8.1
|
|
36
|
+
Requires-Dist: autopep8
|
|
37
|
+
Requires-Dist: gunicorn==23.0.0
|
|
38
|
+
Requires-Dist: pytz==2021.3
|
|
39
|
+
Requires-Dist: sqlparse==0.5.1
|
|
40
|
+
Requires-Dist: toml>=0.10.2
|
|
41
|
+
Requires-Dist: whitenoise==5.3.0
|
|
42
|
+
Requires-Dist: crispy-bootstrap5
|
|
43
|
+
Requires-Dist: django-crispy-forms
|
|
44
|
+
Requires-Dist: django-environ==0.11.2
|
|
45
|
+
Requires-Dist: djangorestframework==3.15.2
|
|
46
|
+
Requires-Dist: djangorestframework-simplejwt==5.3.1
|
|
47
|
+
Requires-Dist: folium
|
|
48
|
+
Requires-Dist: pandas
|
|
49
|
+
Requires-Dist: numpy
|
|
50
|
+
Requires-Dist: orjson
|
|
51
|
+
Requires-Dist: jsonschema
|
|
52
|
+
Requires-Dist: dj-database-url==2.1.0
|
|
34
53
|
|
|
35
54
|
<div align="center">
|
|
36
|
-
<h2>
|
|
37
|
-
<
|
|
38
|
-
<h4>
|
|
55
|
+
<h2>irie</h2>
|
|
56
|
+
<font size="28"><i><b>i</b>nfrastructure <b>r</b>el<b>i</b>ability <b>e</b>ngine</i></font>
|
|
57
|
+
<h4>PEER</h4>
|
|
39
58
|
</div>
|
|
40
59
|
|
|
41
60
|
### External Dependencies
|