irie 0.0.41__py3-none-any.whl → 0.0.43__py3-none-any.whl

irie/init/calid.py

@@ -21,16 +21,16 @@ CESMD = {
     "53-2795F":  ("CE24694", "07-LA-5-24.5"       , "Sylmar - I5/14 Interchange Bridge"),
     "55-0225":   ("CE13795", "07-ORA-5-6.62"      , "Capistrano Beach - I5/Via Calif. Bridge"),
 
-    "58-0215":   ("CE01336",                  None, "El Centro - Hwy8/Meloland Overpass [Abandoned]"),
+#   "58-0215":   ("CE01336",                  None, "El Centro - Hwy8/Meloland Overpass [Abandoned]"),
     "47-0048":   ("CE54730",                  None, "Lake Crowley - Hwy 395 Bridge"),
     "04-0228":   ("CE89686",                  None, "Eureka - Samoa Channel Bridge"),
     "04-0236":   ("CE89324",                  None, "Rio Dell - Hwy 101/Painter St. Overpass"),
     "54-0823G":  ("CE23631",                  None, "San Bernardino - I10/215 Interchange Br"),
     "33-0214L":  ("CE58658",                  None, "Hayward - Hwy 580/238 Interchange Bridge"),
-}
+# }
 
 # TODO: Need to get NBI/CGS Data for these 
-{
+# {
 #-------------------------------------
 
     "34-0006":   ("CE58600",                  None, "Oakland - SF Bay Bridge/East: SAS"),
@@ -116,3 +116,86 @@ CESMD = {
 """
 
 CALID = {v[0]: k for k, v in CESMD.items()}
+
+CESMD_LONG_LAT = {
+ '89973': [40.5093, -124.1196],
+ '89324': [40.5031, -124.1009],
+ '89686': [40.8216, -124.1697],
+ '89736': [40.809, -124.1547],
+ '89735': [40.8136, -124.1594],
+ '47315': [36.8619, -121.5796],
+ '23631': [34.065, -117.2962],
+ '58700': [37.8198, -122.4788],
+ '25749': [34.4497, -119.735],
+ '14406': [33.7494, -118.2715],
+ '58632': [37.7982, -122.3774],
+ '58657': [37.8286, -122.2666],
+ '24775': [34.8663, -118.8836],
+ '58601': [37.8198, -122.3429],
+ '24706': [34.5463, -118.1301],
+ '33742': [35.6696, -117.8187],
+ '24704': [34.0368, -118.3762],
+ '24689': [34.1486, -118.1561],
+ '54730': [37.5733, -118.739],
+ '13705': [33.882, -117.5497],
+ '13795': [33.4659, -117.6674],
+ '68065': [38.1205, -122.2801],
+ '57595': [37.4639, -121.9302],
+ '68185': [38.061, -122.2265],
+ '68717': [38.3484, -122.713],
+ '58754': [37.3574, -122.397],
+ '58658': [37.6907, -122.0993],
+ '76741': [39.368, -120.1151],
+ '89708': [40.9563, -124.1161],
+ '24714': [34.2878, -118.4084],
+ '79421': [39.9213, -123.7634],
+ '68778': [38.1157, -122.5054],
+ '58602': [37.8126, -122.3627],
+ '68184': [38.061, -122.2252],
+ '12666': [33.9153, -116.6085],
+ '68682': [38.0414, -122.1238],
+ '12649': [33.9332, -116.9907],
+ '67771': [38.0278, -121.7514],
+ '57748': [37.4176, -121.9738],
+ '58600': [37.8152, -122.3589],
+ '69760': [38.955, -123.1051],
+ '47707': [36.9067, -121.7818],
+ '03679': [32.6891, -117.1538],
+ '24738': [34.2878, -118.865],
+ '36668': [35.7331, -120.2901],
+ '37728': [35.5955, -121.1256],
+ '03731': [32.838, -117.234],
+ '99190': [41.8797, -124.1373],
+ '25324': [34.2428, -119.1905],
+ '58258': [37.9344, -122.4346],
+ '79757': [39.8485, -123.7019],
+ '68322': [38.0441, -122.1229],
+ '58596': [37.505, -122.12],
+ '24694': [34.3349, -118.5084],
+ '79683': [39.2157, -123.7678],
+ '24670': [34.0314, -118.4354],
+ '69671': [38.4347, -123.1017],
+ '79761': [39.8471, -123.6967],
+ '58656': [37.7837, -122.1769],
+ '67972': [38.5728, -121.587],
+ '99710': [41.5174, -124.0307],
+ '23650': [34.2251, -117.4096],
+ '58677': [37.5836, -122.2515],
+ '25758': [34.9331, -119.5941],
+ '79296': [39.4274, -123.8068],
+ '58536': [37.6746, -122.3895],
+ '58501': [37.6708, -122.0885],
+ '58414': [37.584, -122.509],
+ '01571':  [37.34105, -121.8514],
+ '05235':  [33.96843, -117.44815],
+ '25725': [34.2622, -119.231],
+ '58799': [37.6083, -122.1842],
+ '47646': [36.2391, -121.7747],
+ '88638': [40.881, -122.3809],
+ '22503': [34.2419, -116.9775],
+ '88194': [40.7597, -122.3188],
+ '26917': [34.5215, -120.3369],
+ '34715': [35.0387, -118.1696],
+ '14690': [33.7865, -118.0939],
+ '58678': [37.5048, -122.3346]
+}

irie/init/getNBIData.py

@@ -209,7 +209,7 @@ if __name__ == '__main__':
         # bridgeTable = [i for i in bridgeTable if i["STRUCTURE_NUMBER"] in calids]
 
 
-        print(json.dumps(nbi_data, indent=2))
+        # print(json.dumps(nbi_data, indent=2))
 
     else:
         headers.update({

irie/init/getNBIData2.py

@@ -0,0 +1,304 @@
+#===----------------------------------------------------------------------===#
+#
+#         STAIRLab -- STructural Artificial Intelligence Laboratory
+#
+#===----------------------------------------------------------------------===#
+#
+# First run without arguments to generate list of structure numbers and
+# save to JSON. Then run with that JSON as argv[1] to pull inventory data.
+#
+# Adapted from:
+# https://github.com/psychogeekir/ScrapeNBIBridgeInfo/raw/master/getNBIClimateData.py
+#
+# Claudio M. Perez
+#
+# TODO:
+#
+# - Add option for "SELECTED_TAB": "NBETab", 
+#
+# - Perhaps add something like:
+#     --filter-calid calids.txt
+#   This will be useful for testing, eg, (chrystal's first version)
+#     python getNBIData.py yearly.json --filter-calid <(echo "33 0214L")
+#   
+#   or
+#     python getNBIData.py | python getNBIData.py /dev/stdin <(echo "33 0214L")
+#
+import sys
+import json
+import requests
+from tqdm import tqdm
+from pathlib import Path
+
+NAME = Path(__file__).name
+
+EXAMPLES= """"
+Examples:
+    run to obtain structure numbers list:
+        $ {NAME}
+    run to obtain data from JSON list:
+        $ {NAME} [structure_numbers]
+    run filtering for given structure 33 0214L:
+        $ {NAME} [structure_numbers] --filter-calid "33 0214L"
+    run filtering for given structures:
+        $ {NAME} [structure_numbers] --filter-list [list]
+"""
+
+HELP=f"""
+
+usage:  $ {NAME}
+        $ {NAME} --help
+        $ {NAME} [structure_numbers] [options]
+
+Two-step process to obtain data from the National Bridge Inventory.
+
+Positional Arguments:
+
+Options:
+    -fc, --filter-calid             filter for specific structure.
+    -fl, --filter-list              filter for specified structures in txt file.
+    -h, --help                      print this message and exit.
+
+{EXAMPLES}
+"""
+
+def getBridgeList(headers, start_page=1, totalpages=3, pagesize=10, totalbridges=24, page_nums=None, **kwds):
+    url = 'https://infobridge.fhwa.dot.gov/Data/GetAllBridges'
+
+    payload = {
+        "isShowBridgesApplied":True,
+        "gridParam": {
+            "isShowBridgesApplied":True, "IsFilterApplied":False, "SelectedFilters":None,
+            "SortOrder":"asc", "SortIndex": "STATE_CODE",
+        }
+    }
+
+    pages = []
+    def filter(row):
+        return int(row["STATE_CODE"]) > 0
+
+    if page_nums is None:
+        page_nums = range(start_page, totalpages + 1)
+
+    for pageno in page_nums:
+        try:
+            payload["gridParam"]["PageNumber"] = pageno
+            payload["gridParam"]["PageSize"]   = pagesize
+
+            r = requests.post(url, headers=headers, data=json.dumps(payload))
+
+            if r.status_code == 200:
+                try:
+                    resp = json.loads(eval(r.content.decode('utf-8'))) # [1:-1].replace("\\", ""))
+                except:
+                    print(f"Failed to get page {pageno}", file=sys.stderr)
+                    continue
+
+                bridges = [
+                    {'BRIDGE_YEARLY_ID': row['BRIDGE_YEARLY_ID'], 
+                     'STRUCTURE_NUMBER': row['STRUCTURE_NUMBER'].strip()}
+                        for row in resp["Results"]["rows"] # if filter(row)
+                ]
+                pages.extend(bridges)
+                print(pageno, len(pages), len(bridges), resp["Results"]["rows"][-1]["STATE_NAME"], file=sys.stderr)
+
+        except KeyboardInterrupt:
+            break
+
+    return pages
+
+
+def getNBIData(headers, bridgeTable, years, tab="NBI",
+               keep_query=False):
+
+    url = 'https://infobridge.fhwa.dot.gov/Data/getBridgeInformation'
+
+    _headers = headers.copy()
+    # payload = {
+    #     "requestModel": {
+    #         "SELECTED_TAB": "OverviewTab",
+    #         "SELECTED_YEAR_ID": None,
+    #         "IS_NEW_RECORD": True,
+    #         "IS_YEAR_SELECTED": False,
+    #         "Is_Overview_Bridge_Selected": False,
+    #         "SELECTED_YEAR": None,
+    #         "CURRENT_YEARLY_ID": "25099893",
+    #         "IS_NBI_TREE_SELECTED": False,
+    #         "Folder_Name": None,
+    #         "tabChange": False,
+    #         "BRIDGE_YEARLY_ID": "25099893",
+    #         "NEW_BRIDGE_ID": 58813,
+    #         "SELECTED_NDE_TAB": "General"
+    #     }
+    # }
+
+    payload  = {
+      "requestModel":{
+        "SELECTED_TAB": f"{tab}Tab", 
+        "SELECTED_YEAR_ID": None, 
+        "IS_NEW_RECORD": False,
+        "IS_YEAR_SELECTED": False, 
+        "Is_Overview_Bridge_Selected": False,
+        "NEW_BRIDGE_ID": 0, 
+        "STRUCTURE_NUMBER": None, 
+        "STATE_NAME": None,
+        "STATE_CODE": 0,    
+        "IS_EXPERIMENTAL": False, 
+        "SELECTED_NDE_TAB": "General",
+       #"MERRA_ID": 0,"IS_NBI_TREE_SELECTED": False,"Folder_Name": None,"tabChange": False,
+      }
+    }
+
+
+    referer = 'https://infobridge.fhwa.dot.gov/Data/BridgeDetail/'
+    data = {}
+
+    for i,bridge in enumerate(tqdm(bridgeTable)):
+
+        BRIDGE_YEARLY_ID = bridge['BRIDGE_YEARLY_ID']
+        STRUCTURE_NUMBER = bridge['STRUCTURE_NUMBER']
+
+        # data[STRUCTURE_NUMBER] = []
+
+        for year in years:
+            _headers['referer'] = referer + str(BRIDGE_YEARLY_ID)
+
+            payload["requestModel"].update({
+                  "SELECTED_YEAR":     year,
+                  "CURRENT_YEARLY_ID": BRIDGE_YEARLY_ID,
+                  "BRIDGE_YEARLY_ID":  BRIDGE_YEARLY_ID,
+            })
+
+            r = requests.post(url, data=json.dumps(payload), headers=_headers)
+
+            if r.status_code == 200:
+                htmlcontent = r.content.decode('utf-8')
+                # print(data[STRUCTURE_NUMBER][0])
+                try:
+                    data[STRUCTURE_NUMBER] = [{
+                        k: (
+                            v if k != "Results" else {
+                                kk: vv for kk, vv in v.items() if (kk != "NBIDataQuery" or keep_query)
+                            }
+                        ) for k, v in json.loads(htmlcontent).items()
+                     }]
+                except Exception as e:
+                    print(f">> Error: {e}", file=sys.stderr)
+                    continue
+
+            else:
+                print(f">> Error ({year}) {r.status_code}: {r.content}", file=sys.stderr)
+        
+        if i % 500 == 0:
+            with open(f"nbi_data-{i}.json", "w") as f:
+                json.dump(data, f, indent=2)
+
+    return data
+
+
+if __name__ == '__main__':
+    user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0"
+    request_verification_token = 'CfDJ8M6CuWz5hhxGnmUVXw2yDHQlfeNDzVoF03IbAJ0p3LdaDW7poklPvy74ykYda-qwcrtUXD4rnNzn583Ug7PWbR9IlomGzQh1OQIw_pa9d5TNwdN5p77SDfIfz3yq1nWPzxemEn_8bbh7TGGK9FIwcRY' 
+    cookie = "_ga=GA1.1.478241025.1718907711; _ga_0623JYSC1Q=GS1.1.1718922743.2.0.1718922743.0.0.0; _ga_VW1SFWJKBB=GS1.1.1730789269.3.0.1730789272.0.0.0; _ga_CSLL4ZEK4L=GS1.1.1730789269.3.0.1730789272.0.0.0; _ga_NQ5ZN114SB=GS1.1.1730789269.3.0.1730789272.0.0.0; .AspNetCore.Session=CfDJ8M6CuWz5hhxGnmUVXw2yDHRQxNlIdqc8pBGKOJhMcHphMelhCyOQD7cnzYLVUWcsfCE8KOO8TNogarX5FbmvNQeSW1pTphWgR%2B6RLzPiUWuR4yPiDmb6rg82isfHqoEBhFoziXpFlU2o9pMgQICLsy7WbaeZbSgOl6FTg5Y0vLQ5; __RequestVerificationToken=CfDJ8M6CuWz5hhxGnmUVXw2yDHQXNjHWpjZ61I-CMSrl0yWsdWpCyt2QhUoeZ2L2aY0sqNpGy-wrD8ToMph6-wbfcRPpqORdlVci0ghxWu-3i4PCuWsiOkq90E1WupEYErSXnhsQVwHHGcD63WI7qyXZd7w; _ga_GNYE9X3V7H=GS1.1.1730825963.2.1.1730825988.0.0.0"
+
+    headers = {
+        'authority':       'infobridge.fhwa.dot.gov',
+        'origin':          'https://infobridge.fhwa.dot.gov',
+        'sec-fetch-site':  'same-origin',
+        'sec-fetch-mode':  'cors',
+        'accept-encoding': 'gzip, deflate, br',
+        'accept-language': 'en-US,en;q=0.9',
+
+        '__requestverificationtoken': request_verification_token,
+        'user-agent': user_agent,
+        'cookie': cookie
+    }
+
+    if len(sys.argv) == 1:
+            
+        headers.update({
+            'x-requested-with': 'XMLHttpRequest',
+            'content-type':     'application/json; charset=UTF-8',
+            'accept':           'application/json, text/javascript, */*; q=0.01',
+            'referer':          'https://infobridge.fhwa.dot.gov/Data',
+        })
+
+        bridgeTable = getBridgeList(headers, start_page=511, totalpages=808, pagesize=100)
+        print(json.dumps(bridgeTable, indent=2))
+        with open(f"nbi_codes-california-test.json","w") as f:
+        # with open(f"nbi_codes-california.json","w") as f:
+            json.dump(bridgeTable,f,indent=2)
+        sys.exit()
+
+    elif len(sys.argv) == 2:
+        headers.update({
+                'datatype':     'json',
+                'content-type': 'application/json; charset=UTF-8',
+                'accept':       'application/json, text/plain, */*'
+            })
+        bridgeTable = json.load(open(sys.argv[1]))
+        bridgeTable = [
+            i for i in bridgeTable 
+            if " " in i["STRUCTURE_NUMBER"] and len(i["STRUCTURE_NUMBER"]) in {7, 8}
+            ]
+
+        # calids = list(map(str.strip, open("init/calid.txt").readlines()))
+        # bridgeTable = [i for i in bridgeTable if i["STRUCTURE_NUMBER"] in calids]
+
+        nbi_data = getNBIData(headers, bridgeTable[:], years=(2024,)) #range(2020, 2024))
+        print(json.dumps(nbi_data, indent=2))
+        with open(f"nbi_data-california-test.json","w") as f:
+        # with open(f"nbi_data-california.json","w") as f:
+            json.dump(nbi_data,f,indent=2)
+
+    else:
+        arg1 = sys.argv[1]
+        args2 = iter(sys.argv[2:])
+
+        if arg1 in ["--help", "-h"]:
+            print(HELP)
+            sys.exit
+        else:     
+            headers.update({
+                'datatype':     'json',
+                'content-type': 'application/json; charset=UTF-8',
+                'accept':       'application/json, text/plain, */*'
+            })
+            bridgeTable = json.load(open(sys.argv[1]))
+            for arg in args2:
+                if arg in ["--help", "-h"]:
+                    print(HELP)
+                    sys.exit
+                elif arg in ["--filter-calid", "-fc"]:
+                    calid = next(args2)
+                    bridgeTable = [
+                        i for i in bridgeTable 
+                        if calid in i["STRUCTURE_NUMBER"]
+                    ]
+                    file_ending = calid
+                elif arg in ["--filter-list", "-fl"]:
+                    filename = next(args2)
+                    with open(filename, 'r') as file:
+                        # Check if this is dependent on a specific txt structure (currently 1 bridge/line)
+                        calid = [line.strip() for line in file]     
+                    # print(calid)
+                    bridgeTable = [
+                        i for i in bridgeTable 
+                        if any(j in i["STRUCTURE_NUMBER"] for j in calid)
+                    ]
+                    # print(bridgeTable)
+                    file_ending = Path(filename).name
+
+            # calids = list(map(str.strip, open("init/calid.txt").readlines()))
+            # bridgeTable = [i for i in bridgeTable if i["STRUCTURE_NUMBER"] in calids]
+
+            nbi_data = getNBIData(headers, bridgeTable[:], years=(2024,)) #range(2020, 2024))
+            print(json.dumps(nbi_data, indent=2))
+
+            # TODO: remove this and replace with arg parsing
+            tab = ...
+
+            with open(f"{tab}_data-{file_ending}.json","w") as f:
+            # with open(f"nbi_data-california.json","w") as f:
+                json.dump(nbi_data,f,indent=2)
+

irie/init/management/commands/init_assets.py

@@ -1,6 +1,3 @@
-"""
-python manage.py shell < scripts/make_assets.py
-"""
 import irie
 import lzma
 import tarfile
@@ -47,6 +44,13 @@ SKIP_DESIGN = {
 
 #-----------------------------------
 
+def nbi_reshape(raw):
+    data = defaultdict(dict)
+    for row in raw["Results"]["NBIData"]["NBIDataList"]:
+        data[row["TABLE_NAME"]][row["EXPANDED_FIELD_ALIAS"]] = row["FIELD_VALUE"]
+
+    return dict(data)
+
 
 def load_assets(NBI_DATA):
 
@@ -57,16 +61,12 @@ def load_assets(NBI_DATA):
         return {}
 
     def get_nbi(calid, missing_ok=False):
-        data = defaultdict(dict)
 
         if missing_ok and calid not in NBI_DATA:
             return None
 
-        blocks = NBI_DATA[calid]
-        for row in blocks[-1]["Results"]["NBIData"]["NBIDataList"]:
-            data[row["TABLE_NAME"]][row["EXPANDED_FIELD_ALIAS"]] = row["FIELD_VALUE"]
-
-        return dict(data)
+        blocks = NBI_DATA[calid][-1]
+        return nbi_reshape(blocks)
 
 
     def get_route(bridge):
@@ -101,10 +101,10 @@ def load_assets(NBI_DATA):
     for calid in CESMD: #BRIDGES.values():
         # if "calid" not in bridge:
         try:
-            calid = Asset.objects.get(calid=calid).calid 
+            calid = Asset.objects.get(calid=calid).calid
         except:
             continue
-                
+
         calid = calid.split(" ")[0].replace("-", " ")
         nbi = get_nbi(calid, missing_ok=True)
         if nbi is not None:

irie/init/management/commands/init_predictors.py

@@ -1,5 +1,5 @@
 """
-Assets must already be created
+NOTE: Assets must already be created
 """
 from django.core.management.base import BaseCommand
 from django.core.files.temp import NamedTemporaryFile