irie 0.0.45__py3-none-any.whl → 0.0.47__py3-none-any.whl

irie/init/{getNBIData2.py → getBridgeData.py}

@@ -11,18 +11,9 @@
 # https://github.com/psychogeekir/ScrapeNBIBridgeInfo/raw/master/getNBIClimateData.py
 #
 # Claudio M. Perez
+# Jose S. Yeckle
 #
-# TODO:
 #
-# - Add option for "SELECTED_TAB": "NBETab", 
-#
-# - Perhaps add something like:
-#     --filter-calid calids.txt
-#   This will be useful for testing, eg, (chrystal's first version)
-#     python getNBIData.py yearly.json --filter-calid <(echo "33 0214L")
-#   
-#   or
-#     python getNBIData.py | python getNBIData.py /dev/stdin <(echo "33 0214L")
 #
 import sys
 import json
@@ -32,32 +23,38 @@ from pathlib import Path
 
 NAME = Path(__file__).name
 
-EXAMPLES= """"
+EXAMPLES = f""""
 Examples:
     run to obtain structure numbers list:
         $ {NAME}
-    run to obtain data from JSON list:
-        $ {NAME} [structure_numbers]
+    run to obtain NBI data from JSON list:
+        $ {NAME} <file>
     run filtering for given structure 33 0214L:
-        $ {NAME} [structure_numbers] --filter-calid "33 0214L"
-    run filtering for given structures:
-        $ {NAME} [structure_numbers] --filter-list [list]
+        $ {NAME} <file> --filter-calid "33 0214L"
+    run filtering for file list of given structures:
+        $ {NAME} <file> --filter-list <file> 
+    run to obtain NBI Element data:
+        $ {NAME} <file> --selected-tab "nbe"
+    output prints to standard out. to save the output:
+        $ {NAME} <file> [options] > out.json
 """
 
-HELP=f"""
+HELP = f"""
 
 usage:  $ {NAME}
         $ {NAME} --help
-        $ {NAME} [structure_numbers] [options]
+        $ {NAME} <file> [options]
+        $ {NAME} <file> [options] > out.json
 
-Two-step process to obtain data from the National Bridge Inventory.
+Two-step process to obtain data from the National Bridge Inventory (InfoBridge platform).
 
 Positional Arguments:
 
 Options:
     -fc, --filter-calid             filter for specific structure.
-    -fl, --filter-list              filter for specified structures in txt file.
-    -h, --help                      print this message and exit.
+    -fl, --filter-list              filter for specified structures in plaintext or json file.
+    -t,  --selected-tab             allows to extract different sources of bridge data, currently supports "nbi" and "nbe".
+    -h,  --help                     print this message and exit.
 
 {EXAMPLES}
 """
@@ -108,8 +105,8 @@ def getBridgeList(headers, start_page=1, totalpages=3, pagesize=10, totalbridges
     return pages
 
 
-def getNBIData(headers, bridgeTable, years, tab="NBI",
-               keep_query=False):
+def getBridgeData(headers, bridgeTable, years,
+               keep_query=False,tab="nbi"):
 
     url = 'https://infobridge.fhwa.dot.gov/Data/getBridgeInformation'
 
@@ -134,7 +131,7 @@ def getNBIData(headers, bridgeTable, years, tab="NBI",
 
     payload  = {
       "requestModel":{
-        "SELECTED_TAB": f"{tab}Tab", 
+        "SELECTED_TAB": "NBITab",           # Change this to get other tab information
         "SELECTED_YEAR_ID": None, 
         "IS_NEW_RECORD": False,
         "IS_YEAR_SELECTED": False, 
@@ -148,7 +145,16 @@ def getNBIData(headers, bridgeTable, years, tab="NBI",
        #"MERRA_ID": 0,"IS_NBI_TREE_SELECTED": False,"Folder_Name": None,"tabChange": False,
       }
     }
-
+    
+    if tab.lower() == "nbi":
+        pass
+    elif tab.lower() == "nbe":
+        payload["requestModel"].update({
+            "SELECTED_TAB": "NBETab"
+        })
+    else:
+        print("ERROR: Please select valid tab.")
+        sys.exit()
 
     referer = 'https://infobridge.fhwa.dot.gov/Data/BridgeDetail/'
     data = {}
@@ -189,6 +195,7 @@ def getNBIData(headers, bridgeTable, years, tab="NBI",
             else:
                 print(f">> Error ({year}) {r.status_code}: {r.content}", file=sys.stderr)
         
+        # Every 500 records, save a json as a progress and sanity check
         if i % 500 == 0:
             with open(f"nbi_data-{i}.json", "w") as f:
                 json.dump(data, f, indent=2)
@@ -225,12 +232,15 @@ if __name__ == '__main__':
 
         bridgeTable = getBridgeList(headers, start_page=511, totalpages=808, pagesize=100)
         print(json.dumps(bridgeTable, indent=2))
-        with open(f"nbi_codes-california-test.json","w") as f:
+        # with open(f"nbi_codes-california-test.json","w") as f:
         # with open(f"nbi_codes-california.json","w") as f:
-            json.dump(bridgeTable,f,indent=2)
+            # json.dump(bridgeTable,f,indent=2)
         sys.exit()
 
     elif len(sys.argv) == 2:
+        if sys.argv[1] in ["--help", "-h"]:
+            print(HELP)
+            sys.exit()
         headers.update({
                 'datatype':     'json',
                 'content-type': 'application/json; charset=UTF-8',
@@ -245,11 +255,11 @@ if __name__ == '__main__':
         # calids = list(map(str.strip, open("init/calid.txt").readlines()))
         # bridgeTable = [i for i in bridgeTable if i["STRUCTURE_NUMBER"] in calids]
 
-        nbi_data = getNBIData(headers, bridgeTable[:], years=(2024,)) #range(2020, 2024))
+        nbi_data = getBridgeData(headers, bridgeTable[:], years=(2024,)) #range(2020, 2024))
         print(json.dumps(nbi_data, indent=2))
-        with open(f"nbi_data-california-test.json","w") as f:
-        # with open(f"nbi_data-california.json","w") as f:
-            json.dump(nbi_data,f,indent=2)
+        # with open(f"nbi_data-california-test.json","w") as f:
+        # with open(f"data/nbi_data-california.json","w") as f:
+            # json.dump(nbi_data,f,indent=2)
 
     else:
         arg1 = sys.argv[1]
@@ -257,7 +267,7 @@ if __name__ == '__main__':
 
         if arg1 in ["--help", "-h"]:
             print(HELP)
-            sys.exit
+            sys.exit()
         else:     
             headers.update({
                 'datatype':     'json',
@@ -265,40 +275,45 @@ if __name__ == '__main__':
                 'accept':       'application/json, text/plain, */*'
             })
             bridgeTable = json.load(open(sys.argv[1]))
+            selected_tab="nbi"
             for arg in args2:
                 if arg in ["--help", "-h"]:
                     print(HELP)
-                    sys.exit
+                    sys.exit()
+                elif arg in ["--selected-tab","-t"]:
+                    selected_tab=next(args2)
                 elif arg in ["--filter-calid", "-fc"]:
                     calid = next(args2)
                     bridgeTable = [
                         i for i in bridgeTable 
                         if calid in i["STRUCTURE_NUMBER"]
                     ]
-                    file_ending = calid
                 elif arg in ["--filter-list", "-fl"]:
-                    filename = next(args2)
-                    with open(filename, 'r') as file:
-                        # Check if this is dependent on a specific txt structure (currently 1 bridge/line)
-                        calid = [line.strip() for line in file]     
+                    filter_file = next(args2)
+                    with open(filter_file, 'r') as file:
+                        # print(file)
+                        extension = Path(filter_file).suffix
+                        # if json, load as json
+                        if extension==".json":
+                            calid = json.load(file)
+                        # all other file extensions, including
+                        # .txt and none, should be plaintext
+                        # and will be read line-by-line
+                        else: 
+                            calid = [line.strip() for line in file]
                     # print(calid)
                     bridgeTable = [
                         i for i in bridgeTable 
                         if any(j in i["STRUCTURE_NUMBER"] for j in calid)
                     ]
                     # print(bridgeTable)
-                    file_ending = Path(filename).name
 
             # calids = list(map(str.strip, open("init/calid.txt").readlines()))
             # bridgeTable = [i for i in bridgeTable if i["STRUCTURE_NUMBER"] in calids]
 
-            nbi_data = getNBIData(headers, bridgeTable[:], years=(2024,)) #range(2020, 2024))
-            print(json.dumps(nbi_data, indent=2))
-
-            # TODO: remove this and replace with arg parsing
-            tab = ...
-
-            with open(f"{tab}_data-{file_ending}.json","w") as f:
-            # with open(f"nbi_data-california.json","w") as f:
-                json.dump(nbi_data,f,indent=2)
+            bridge_data = getBridgeData(headers, bridgeTable[:], years=(2024,), tab=selected_tab) #range(2020, 2024))
+            print(json.dumps(bridge_data, indent=2))
+            # with open(f"nbi_data-california-test.json","w") as f:
+            # with open(f"data/{selected_tab}_data-california.json","w") as f:
+                # json.dump(bridge_data,f,indent=2)
 

irie/init/getNBIData.py

@@ -185,7 +185,7 @@ if __name__ == '__main__':
     }
 
     if len(sys.argv) == 1:
-            
+
         headers.update({
             'x-requested-with': 'XMLHttpRequest',
             'content-type':     'application/json; charset=UTF-8',

irie/init/management/commands/init_cesmd.py

@@ -10,6 +10,8 @@ DATA = Path(irie.__file__).parents[0]/"init"/"data"
 
 with open(DATA/"cgs_data.json") as f:
     CGS_DATA = json.loads(f.read())
+with open(DATA/"nbi_data_cesmd.json") as f:
+    NBI_DATA = json.loads(f.read())
 
 class Command(BaseCommand):
     def handle(self, *args, **kwargs):
@@ -29,6 +31,7 @@ class Command(BaseCommand):
                 asset.cesmd = cesmd 
                 asset.calid = calid
                 asset.cgs_data = CGS_DATA.get(cesmd, {})
+                asset.nbi_data = NBI_DATA.get(calid.replace("-", " "), {})
                 
                 asset.save()
                 print(asset)

{irie-0.0.45.dist-info → irie-0.0.47.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: irie
-Version: 0.0.45
+Version: 0.0.47
 Summary: An infrastructure resilience engine
 Author-email: "Claudio M. Perez" <50180406+claudioperez@users.noreply.github.com>
 Project-URL: Repository, https://github.com/STAIRLab

{irie-0.0.45.dist-info → irie-0.0.47.dist-info}/RECORD

@@ -59,13 +59,13 @@ irie/apps/inventory/__init__.py,sha256=wkkNngGxgYcCM745-rlvP6ynrI0b0QN3aWmLWDsR8
 irie/apps/inventory/admin.py,sha256=e_W8ls_ARLLf8rn3oJd-YkPdTGMIrcN-mRrxk2BcY3c,472
 irie/apps/inventory/apps.py,sha256=bZ6qYIwPMG4_4IeLfg9N4WuZAEgEVj84oOswV-7_MAI,424
 irie/apps/inventory/fields.py,sha256=J3nTImPsuCeiOWBizSL4tnuKs36sPfXALNTKEZY-wVg,79
-irie/apps/inventory/filters.py,sha256=ZEM4aubBqATThyaUhdFFoZnZBUASGj5pwtsXcgyOz8Q,2295
+irie/apps/inventory/filters.py,sha256=P0L1AFCcK8X8NWLn0tBVVfEdtCHqzTWUPTeUGflGdk0,2539
 irie/apps/inventory/forms.py,sha256=y8tcIGInXDg7KCf1OWd1jtc4umJsm8rf8-4O8nDhNd4,1244
 irie/apps/inventory/models.py,sha256=ie7XhFDujpoyIinmX-YfScVx28SkkRhFadU0iNLC9HI,6073
 irie/apps/inventory/sitemaps.py,sha256=Nha1MTsIH_ad7JyoxwonPytp7MNuEhDNszkEUOmlN0o,826
 irie/apps/inventory/tables.py,sha256=vZdPOcbN1ibuWXqLwbBUoQw9iavwa1GJ5fd83k8bu7Y,27874
 irie/apps/inventory/urls.py,sha256=u2LNLB4K4fDq3yPYtREKf3i7_yHRzrTP3cb22G8Brvk,1578
-irie/apps/inventory/views.py,sha256=t-RgtEhSkv1VrvO3d2tvytp81TTlmObQcB2Y04w8zg8,21234
+irie/apps/inventory/views.py,sha256=cjL2v4_7q2S7OczeNdh_g7pVxXdkKgaqj60oQaOEED8,21422
 irie/apps/inventory/archive/CESMD.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 irie/apps/inventory/archive/calid.py,sha256=3L8MbPIGOE3kzDnqeyY055pRBiF2O2l0cmpuDbTsdTg,3014
 irie/apps/inventory/migrations/0001_initial.py,sha256=PwTHv4Q3gqWFha--8Zp9kUOh-cYalB14jXj7RVJUVnw,1786
@@ -398,7 +398,7 @@ irie/apps/templates/inventory/asset-event-summary.html,sha256=5G7uZVf7kVL0wBaAfs
 irie/apps/templates/inventory/asset-on-map.html,sha256=LsnvUmTi6TkHSv9sZ_L4KMOwipP_4fkQXDqLUjLS0uo,17062
 irie/apps/templates/inventory/asset-profile.html,sha256=oJiHwQ7ilmRUQy31PMXJgZUPbefJKBNWhjV6IF8UgR0,13576
 irie/apps/templates/inventory/asset-sensors.html,sha256=qLnlhnmOeT5uL1xH67mFPOgufXV_PJ31KEx5Aqt0TTw,2406
-irie/apps/templates/inventory/asset-table.html,sha256=hHjZqBBE7FfJPUxVAfpfD4w4C3KiJCdr-rEQHnMbDAE,13204
+irie/apps/templates/inventory/asset-table.html,sha256=QAziJ_VKc_BOmJ6YCrNbm_QUd9qqMwFZsWNieIVLlq4,13202
 irie/apps/templates/inventory/bridge-dashboard.html,sha256=67zrDlih3LOi9xFVwPVZZkgRT3DO-lE_qRL7q8Uz9GY,45472
 irie/apps/templates/inventory/bridge.html,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 irie/apps/templates/inventory/dashboard.html,sha256=_qhFiPP6xN-UzZj5NuM5e9NxsgDHJkQ89oXN1cBwhj0,7505
@@ -466,28 +466,31 @@ irie/init/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 irie/init/__main__.py,sha256=waUELqnhqZirROMIb5wMzD4mK5fgrUvWRsXoTXhjtxY,807
 irie/init/bridges.py,sha256=OBBKnLx2lmMKDfeR0C8CBSbnpnzO0lfokBypD2nEFcM,74198
 irie/init/calid.py,sha256=-861tihqB-u2ZfXWqAW6RzgSmR1UvcSX4Y_ZzfJVWMw,10883
+irie/init/getBridgeData.py,sha256=PU4OcdXCR0D-CCBX_T3PjyYL0QJKYyz2I4AMhaYKaZk,12650
 irie/init/getCGSData.py,sha256=iZG3Ab1Y_rhiliKCPNy0MZrKBsfEe6ShgSFz2ttvTUU,2916
 irie/init/getCGSevents.py,sha256=4dBkFFJQrUs6gfl1Nj-_R2UOZj0B_T017a6tC7fUHOw,421
-irie/init/getNBIData.py,sha256=qCd2377wt1MmRuMm-mgRzUxQGKQBYwhMxwxyRLhVe-k,9404
-irie/init/getNBIData2.py,sha256=JIbdG2iMq9mUNXBlw_3j_lr71wk7ScVgSWYZOUdZBvo,11782
+irie/init/getNBIData.py,sha256=LJ3GycQ_0A-TbevlRr4zplwz3KDJHZFK1WGEK1hufoU,9392
 irie/init/hayward.zip,sha256=WECOnYLP_3EeKHQzdxm17MHTncF2QdgXI28Vm6Iopt0,124675
 irie/init/data/cgs-stations.json,sha256=h-KKF-u31j_OQnQZQlmFzDjqgi0AcsPNzohn7bCNq0Q,105563
 irie/init/data/cgs_data.json,sha256=l2lADALMXVLKY6jn-rvEZjYWcZjGHLk22ZudocdR81Q,27752
+irie/init/data/nbe_data-california.json,sha256=1bUojfq0ES98O8G606jPOx4wSox_GxTkwkj_SgG6ymo,8101291
+irie/init/data/nbi_codes-california-filtered.json,sha256=FgpvLdc7X1O27H-35ZPYyxKKfyQ87rW5Onpmtp51OWw,814
 irie/init/data/nbi_codes-california.json,sha256=UPFbBzxY2NshREecsP4evCA8vdvXFRvG4rYT4_KHmQs,2398694
 irie/init/data/nbi_data-california.json,sha256=nyslZbMf30WrtdyqCpMvxd-NNxNzFHtZX_TDWRZFob0,8534365
+irie/init/data/nbi_data_cesmd.json,sha256=6FkK7dOx3oK7-_ew0jS4F2owRH-tueiE-y7kU9V_Png,11234353
 irie/init/data/nbi_definitions.json,sha256=NS3yNP1i1E_JXfGW_I_xOoNRsfeljDA5BZ_1h0Cu_1I,4301
 irie/init/data/nbi/04.tar,sha256=ubRgxWBshe0cNKxftbEVyFeh1HcUmFarVL2BBHo9Tyk,1884160
 irie/init/data/networks/soga_corridors.json,sha256=AQOUaKGNWkQxQKLfqphE9Qb_-NRmjN9IwK59SGjutxE,45496
 irie/init/management/commands/init_assets.py,sha256=z-QczBwa7altFy8oLgKtF2_ogHwWOGGFiMJYLkol-zU,5709
-irie/init/management/commands/init_cesmd.py,sha256=00aNGLN6M_oDhMyKySJuqaPW0IwJal3TxsxCKn1lCjs,1200
+irie/init/management/commands/init_cesmd.py,sha256=CuttXboaYoIHAlWnsfE-h6cdCQqoO0iVGI11F91i-uo,1355
 irie/init/management/commands/init_corridors.py,sha256=EzLk0HUiFxlco-2u0rypewOc9mAo_raqXC2_UCG8a_w,1241
 irie/init/management/commands/init_predictors.py,sha256=edCf7d36cwl4YitXLtvOfsPee-BiPF55_Gr73OvTPcU,2368
 irie/init/management/commands/make_asset.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 irie/pull/nbi.py,sha256=KpBjJ9GEU72Qk1t4bGN9cg0QBeVJ8k9XcI3Y1oSgIR0,11478
 irie/rest/__main__.py,sha256=6Nf_-Rr9zGmMyp_wqCFDO7ls9QPnPd9UyUgN17rIGYw,3680
 irie/usgs/__main__.py,sha256=HiSvPn5IW5IqRiCk1qRRq5dCy3-7iISw7v1P_w2rLrk,5049
-irie-0.0.45.dist-info/METADATA,sha256=Sfs8DI6WGsfvEHkGCAHCp256LDnelpkFeHsEBkVOVLc,3207
-irie-0.0.45.dist-info/WHEEL,sha256=tTnHoFhvKQHCh4jz3yCn0WPTYIy7wXx3CJtJ7SJGV7c,91
-irie-0.0.45.dist-info/entry_points.txt,sha256=A_3h9wPBGfxGQ78_MGa-nO6Z0foxOYeAnIE47jxEztg,44
-irie-0.0.45.dist-info/top_level.txt,sha256=zVCxi5E2nkISZPzKq8VEhWe_dGuPcefLYV1tYqQdwlY,5
-irie-0.0.45.dist-info/RECORD,,
+irie-0.0.47.dist-info/METADATA,sha256=UY4aOzGLBIxZib3YMGJmtUSiMQbiZzMNoYlWIFaPPuY,3207
+irie-0.0.47.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+irie-0.0.47.dist-info/entry_points.txt,sha256=A_3h9wPBGfxGQ78_MGa-nO6Z0foxOYeAnIE47jxEztg,44
+irie-0.0.47.dist-info/top_level.txt,sha256=zVCxi5E2nkISZPzKq8VEhWe_dGuPcefLYV1tYqQdwlY,5
+irie-0.0.47.dist-info/RECORD,,

{irie-0.0.45.dist-info → irie-0.0.47.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (77.0.1)
+Generator: setuptools (77.0.3)
 Root-Is-Purelib: true
 Tag: py3-none-any