NSI Integration

This is an example to demonstrate how BrailsPlusPlus integrates with the National Structures Inventory. There are two ways to integrate:

  1. Create an inventory that does not include footprint information directly from NSI.

  2. Integrate an existing AssetInventory with NSI. For each Asset in the asset inventory, the features from NSI are obtained and merged with the asset’s existing features. This is done by finding an asset in the NSI located in the footprint.

  1# written: fmk, bacetiner
  2# Last updated: 11-06-2024
  3# License: BSD-2
  4
  5"""
  6brails_nsi.py
  7=============
  8Example demonstrating the use of NSI scraper module .
  9
 10 Purpose: Testing 1) get_class method of Importer
 11                  2) get_raw_data_given_boundary and
 12                     get_filtered_data_given_inventory methods of NSI_Parser
 13                  3) get_footprints method from scraper
 14                  4) print_info and write_to_geojson method of AssetInventory
 15                     objects
 16"""
 17
 18import argparse
 19import sys
 20import os
 21import json
 22
 23# the following line is not neeeded if brails is imported from pypi
 24#   .. it is included here as it allows us to test the code on a nightly basis
 25sys.path.insert(1, "../../")
 26
 27import brails
 28
 29from brails import Importer
 30
 31def main():
 32    
 33    # Create the argument parser
 34    parser = argparse.ArgumentParser(description="Demonstrate Importer.")
 35    parser.add_argument('scraper', type=str, help="Footprint Scraper")
 36    parser.add_argument('location', type=str, help="Location")
 37
 38    # Parse the arguments
 39    args = parser.parse_args()
 40
 41    # Create an importer to get the required classes:
 42    importer = Importer()
 43    
 44    # Select a region and create its RegionBoundary:
 45    region_boundary_class = importer.get_class('RegionBoundary')
 46    region_boundary_object = region_boundary_class({'type': 'locationName', 'data': args.location})
 47
 48    #
 49    # Use NSI_Parser to get the NSI raw data for the specified region
 50    #
 51    
 52    nsi_class = importer.get_class('NSI_Parser')
 53    nsi = nsi_class()
 54    nsi_inventory = nsi.get_raw_data_given_boundary(region_boundary_object, 'ft')
 55    print(f'Total number of assets detected using NSI is {len(nsi_inventory.inventory)}')
 56
 57    print('\n******** SMALL NSI INVENTORY  *********')    
 58    nsi_inventory_subset = nsi_inventory.get_random_sample(2, 200)    
 59    nsi_inventory_subset.print_info()        
 60    
 61    #
 62    # Now Use the FootprintScraper to get footprints and integrate NSI inventory data
 63    #   - will integrate on smaller inventory set
 64    #
 65
 66
 67    scraper_class = importer.get_class(args.scraper)
 68    scraper = scraper_class({'length': 'ft'})
 69    scraper_inventory = scraper.get_footprints(region_boundary_object)
 70    print(f'Total number of assets detected using {args.scraper} is {len(scraper_inventory.inventory)}')    
 71    # Create inventories that are a small subset of the obtained footprints & print:
 72    inventory_subset1 = scraper_inventory.get_random_sample(5, 200)
 73    inventory_subset2 = scraper_inventory.get_random_sample(5, 200)    
 74    
 75    print('\n******** SMALL INVENTORY WITH NO NSI_DATA *********')    
 76    inventory_subset1.print_info()
 77    
 78    # integrate subset with raw and filtered NSI data
 79    subset_nsi_raw_data = nsi.get_raw_data_given_inventory(inventory_subset1, 'ft')
 80    subset_nsi_processed_data = nsi.get_filtered_data_given_inventory(inventory_subset2, 'ft')
 81    
 82    #
 83    # Print the inventories
 84    #
 85    
 86    print('\n******** SMALL INVENTORY WITH RAW NSI DATA *********')
 87    subset_nsi_raw_data.print_info()
 88
 89    print('\n******** SMALL INVENTORY WITH PROCESSES NSI DATA*********')
 90    subset_nsi_processed_data.print_info()
 91
 92    #
 93    # Write the extracted inventory to a GeoJSON file:
 94    #
 95    
 96    FILE_PATH = 'tmp/smallinv.geojson'
 97    directory = os.path.dirname(FILE_PATH)
 98    os.makedirs(directory, exist_ok=True)
 99    geojson = subset_nsi_processed_data.write_to_geojson(FILE_PATH)
100
101    
102# Run the main function if this script is executed directly
103if __name__ == "__main__":
104    main()

The script is executed by entering the following command in a terminal window:

python3 brails_nsi.py OSM_FootprintScraper "Berkeley, CA"

As shown in the print_output() of the smaller NSI inventory, the coordinates for such an inventory only contain points instead of the building footprints.

1******** SMALL NSI INVENTORY  *********
2AssetInventory
3Inventory stored in:  dict
4Key:  474740175 Asset:
5	 Coordinates:  [[-122.262559, 37.879117]]
6	 Features:  {'fd_id': 474740175, 'bid': '849VVPHP+JXW-2-1-2-2', 'occtype': 'RES1-2SWB', 'st_damcat': 'RES', 'bldgtype': 'M', 'found_type': 'B', 'cbfips': '060014225001003', 'pop2amu65': 2, 'pop2amo65': 1, 'pop2pmu65': 2, 'pop2pmo65': 1, 'sqft': 1773, 'num_story': 2, 'ftprntid': '06001_559590', 'ftprntsrc': 'NGA', 'students': 0, 'found_ht': 2, 'val_struct': 359933.479, 'val_cont': 179966.739, 'val_vehic': 27000, 'source': 'P', 'med_yr_blt': 1939, 'firmzone': 'NA', 'o65disable': 0.22, 'u65disable': 0.03, 'x': -122.2625592, 'y': 37.879116732, 'ground_elv': 350.8512103598022, 'ground_elv_m': 106.93944549560547, 'type': 'Building'}
7Key:  475206003 Asset:
8	 Coordinates:  [[-122.279912, 37.894081]]
9	 Features:  {'fd_id': 475206003, 'bid': '849VVPVC+J2P-4-4-4-3', 'occtype': 'RES1-2SNB', 'st_damcat': 'RES', 'bldgtype': 'W', 'found_type': 'C', 'cbfips': '060014213001020', 'pop2amu65': 2, 'pop2amo65': 0, 'pop2pmu65': 1, 'pop2pmo65': 0, 'sqft': 2341, 'num_story': 2, 'ftprntid': '06001_730116', 'ftprntsrc': 'NGA', 'students': 0, 'found_ht': 1.5, 'val_struct': 378037.886, 'val_cont': 189018.943, 'val_vehic': 27000, 'source': 'P', 'med_yr_blt': 1939, 'firmzone': 'NA', 'o65disable': 0.22, 'u65disable': 0.03, 'x': -122.2799116, 'y': 37.894081399, 'ground_elv': 195.71405070861817, 'ground_elv_m': 59.65364074707031, 'type': 'Building'}

The output also shows that the number of buildings in the two inventories, nsi_inventory and scraper_inventory, are different:

Total number of assets detected using NSI is 27705
Total number of assets detected using OSM_FootprintScraper is 35547

As there are different numbers of buildings in the two inventories, when integrating NSI dataset into the footprint inventory, there will be assets for which no NSI data exists. In the merge performed with the example run, 2 out of the 5 assets in the subset inventories do not have data available in NSI. This is shown in the output lines:

The original and integrated inventories are as shown:

 1******** SMALL INVENTORY WITH NO NSI_DATA *********
 2AssetInventory
 3Inventory stored in:  dict
 4Key:  2989 Asset:
 5	 Coordinates:  [[-122.2539358, 37.8823889], [-122.2538841, 37.8822577], [-122.2538636, 37.882258], [-122.2538454, 37.8821659], [-122.2540379, 37.8821623], [-122.2540337, 37.8822459], [-122.2539784, 37.882245], [-122.2540264, 37.8823667], [-122.2539358, 37.8823889]]
 6	 Features:  {'type': 'Building', 'buildingheight': 'NA', 'erabuilt': 'NA', 'numstories': 1, 'roofshape': 'NA', 'fpAreas': 2973}
 7Key:  13333 Asset:
 8	 Coordinates:  [[-122.2743673, 37.8721826], [-122.2743469, 37.8720391], [-122.2743897, 37.8720353], [-122.2743867, 37.8720143], [-122.2744387, 37.8720097], [-122.2744416, 37.8720305], [-122.2744612, 37.8720287], [-122.2744816, 37.8721725], [-122.2743673, 37.8721826]]
 9	 Features:  {'type': 'Building', 'buildingheight': 'NA', 'erabuilt': 'NA', 'numstories': 'NA', 'roofshape': 'NA', 'fpAreas': 1866}
10Key:  1989 Asset:
11	 Coordinates:  [[-122.2534638, 37.856443], [-122.2534705, 37.856479], [-122.2535403, 37.8564711], [-122.253547, 37.8564912], [-122.2535946, 37.8565177], [-122.2535993, 37.8565277], [-122.2536127, 37.8565272], [-122.2535939, 37.8564404], [-122.253494, 37.8564531], [-122.2534913, 37.8564399], [-122.2534638, 37.856443]]
12	 Features:  {'type': 'Building', 'buildingheight': 'NA', 'erabuilt': 'NA', 'numstories': 'NA', 'roofshape': 'NA', 'fpAreas': 625}
13Key:  9361 Asset:
14	 Coordinates:  [[-122.2654248, 37.8550534], [-122.2654134, 37.854993], [-122.2654737, 37.8549859], [-122.2654851, 37.8550463], [-122.2654248, 37.8550534]]
15	 Features:  {'type': 'Building', 'buildingheight': 'NA', 'erabuilt': 'NA', 'numstories': 'NA', 'roofshape': 'NA', 'fpAreas': 392}
16Key:  17479 Asset:
17	 Coordinates:  [[-122.292788, 37.8763201], [-122.2927717, 37.8762696], [-122.2928486, 37.8762541], [-122.2928649, 37.8763047], [-122.292788, 37.8763201]]
18	 Features:  {'type': 'Building', 'buildingheight': 'NA', 'erabuilt': 'NA', 'numstories': 'NA', 'roofshape': 'NA', 'fpAreas': 435}
19
20Getting National Structure Inventory (NSI) building data for the entered location input...
21Found a total of 3 building points in NSI that are within the entered region of interest
22keys:  [2989, 13333, 1989, 9361, 17479]  index    [0, 1, 2]
23
24Getting National Structure Inventory (NSI) building data for the entered location input...
25Found a total of 3 building points in NSI that match the footprint data.
26keys:  [2989, 13333, 1989, 9361, 17479]  index    [0, 1, 2]
27
28******** SMALL INVENTORY WITH RAW NSI DATA *********
29AssetInventory
30Inventory stored in:  dict
31Key:  2989 Asset:
32	 Coordinates:  [[-122.2539358, 37.8823889], [-122.2538841, 37.8822577], [-122.2538636, 37.882258], [-122.2538454, 37.8821659], [-122.2540379, 37.8821623], [-122.2540337, 37.8822459], [-122.2539784, 37.882245], [-122.2540264, 37.8823667], [-122.2539358, 37.8823889]]
33	 Features:  {'type': 'Building', 'buildingheight': 'NA', 'erabuilt': 1954, 'numstories': 1, 'roofshape': 'NA', 'fpAreas': 2973, 'fd_id': 475445345, 'bid': '849VVP4W+HHW-2-2-2-2', 'occtype': 'IND6', 'st_damcat': 'IND', 'bldgtype': 'W', 'found_type': 'S', 'cbfips': '060014239021001', 'pop2amu65': 0, 'pop2amo65': 0, 'pop2pmu65': 4, 'pop2pmo65': 0, 'sqft': 6506.02409, 'num_story': 1, 'ftprntid': '06001_548222', 'ftprntsrc': 'NGA', 'students': 0, 'found_ht': 0.5, 'val_struct': 958384.635, 'val_cont': 958384.635, 'val_vehic': 81000, 'source': 'E', 'med_yr_blt': 1939, 'firmzone': 'NA', 'o65disable': 0.22, 'u65disable': 0.03, 'x': -122.2535494, 'y': 37.85648153, 'ground_elv': 239.3753899307251, 'ground_elv_m': 72.96161651611328, 'lon': -122.2539358, 'lat': 37.882252355, 'fparea': 1592, 'repaircost': 389506.045, 'constype': 'RM1', 'occupancy': 'RES1'}
34Key:  13333 Asset:
35	 Coordinates:  [[-122.2743673, 37.8721826], [-122.2743469, 37.8720391], [-122.2743897, 37.8720353], [-122.2743867, 37.8720143], [-122.2744387, 37.8720097], [-122.2744416, 37.8720305], [-122.2744612, 37.8720287], [-122.2744816, 37.8721725], [-122.2743673, 37.8721826]]
36	 Features:  {'type': 'Building', 'buildingheight': 'NA', 'erabuilt': 1992, 'numstories': 1, 'roofshape': 'NA', 'fpAreas': 1866, 'fd_id': 473584913, 'bid': '849VVPJW+WC4-6-2-5-3', 'occtype': 'RES1-1SWB', 'st_damcat': 'RES', 'bldgtype': 'M', 'found_type': 'B', 'cbfips': '060014216002007', 'pop2amu65': 2, 'pop2amo65': 1, 'pop2pmu65': 2, 'pop2pmo65': 1, 'sqft': 1592, 'num_story': 1, 'ftprntid': '06001_813170', 'ftprntsrc': 'NGA', 'students': 0, 'found_ht': 2, 'val_struct': 389506.045, 'val_cont': 194753.022, 'val_vehic': 27000, 'source': 'P', 'med_yr_blt': 1954, 'firmzone': 'NA', 'o65disable': 0.22, 'u65disable': 0.03, 'x': -122.2539358, 'y': 37.882252355, 'ground_elv': 1004.1034449096679, 'ground_elv_m': 306.05072021484375, 'lon': -122.2743974, 'lat': 37.872097621, 'fparea': 1539.29, 'repaircost': 320524.48, 'constype': 'W1', 'occupancy': 'RES3A'}
37Key:  1989 Asset:
38	 Coordinates:  [[-122.2534638, 37.856443], [-122.2534705, 37.856479], [-122.2535403, 37.8564711], [-122.253547, 37.8564912], [-122.2535946, 37.8565177], [-122.2535993, 37.8565277], [-122.2536127, 37.8565272], [-122.2535939, 37.8564404], [-122.253494, 37.8564531], [-122.2534913, 37.8564399], [-122.2534638, 37.856443]]
39	 Features:  {'type': 'Building', 'buildingheight': 'NA', 'erabuilt': 1939, 'numstories': 1, 'roofshape': 'NA', 'fpAreas': 625, 'fd_id': 474705763, 'bid': '849VVPCG+R6Q-4-2-3-2', 'occtype': 'RES3A', 'st_damcat': 'RES', 'bldgtype': 'W', 'found_type': 'S', 'cbfips': '060014223003010', 'pop2amu65': 1, 'pop2amo65': 0, 'pop2pmu65': 0, 'pop2pmo65': 0, 'sqft': 1539.29, 'num_story': 1, 'ftprntid': '06001_34778', 'ftprntsrc': 'NGA', 'students': 0, 'found_ht': 0.5, 'val_struct': 320524.48, 'val_cont': 160262.24, 'val_vehic': 27000, 'source': 'P', 'med_yr_blt': 1992, 'firmzone': 'NA', 'o65disable': 0.22, 'u65disable': 0.03, 'x': -122.2743974, 'y': 37.872097621, 'ground_elv': 165.9004509815979, 'ground_elv_m': 50.56645584106445, 'lon': -122.2535494, 'lat': 37.85648153, 'fparea': 6506.02409, 'repaircost': 958384.635, 'constype': 'W1', 'occupancy': 'IND6'}
40Key:  9361 Asset:
41	 Coordinates:  [[-122.2654248, 37.8550534], [-122.2654134, 37.854993], [-122.2654737, 37.8549859], [-122.2654851, 37.8550463], [-122.2654248, 37.8550534]]
42	 Features:  {'type': 'Building', 'buildingheight': 'NA', 'erabuilt': 'NA', 'numstories': 'NA', 'roofshape': 'NA', 'fpAreas': 392}
43Key:  17479 Asset:
44	 Coordinates:  [[-122.292788, 37.8763201], [-122.2927717, 37.8762696], [-122.2928486, 37.8762541], [-122.2928649, 37.8763047], [-122.292788, 37.8763201]]
45	 Features:  {'type': 'Building', 'buildingheight': 'NA', 'erabuilt': 'NA', 'numstories': 'NA', 'roofshape': 'NA', 'fpAreas': 435}
46
47******** SMALL INVENTORY WITH PROCESSES NSI DATA*********
48AssetInventory
49Inventory stored in:  dict
50Key:  2989 Asset:
51	 Coordinates:  [[-122.2539358, 37.8823889], [-122.2538841, 37.8822577], [-122.2538636, 37.882258], [-122.2538454, 37.8821659], [-122.2540379, 37.8821623], [-122.2540337, 37.8822459], [-122.2539784, 37.882245], [-122.2540264, 37.8823667], [-122.2539358, 37.8823889]]
52	 Features:  {'type': 'Building', 'buildingheight': 'NA', 'erabuilt': 1954, 'numstories': 1, 'roofshape': 'NA', 'fpAreas': 2973, 'fd_id': 475445345, 'bid': '849VVP4W+HHW-2-2-2-2', 'occtype': 'IND6', 'st_damcat': 'IND', 'bldgtype': 'W', 'found_type': 'S', 'cbfips': '060014239021001', 'pop2amu65': 0, 'pop2amo65': 0, 'pop2pmu65': 4, 'pop2pmo65': 0, 'sqft': 6506.02409, 'num_story': 1, 'ftprntid': '06001_548222', 'ftprntsrc': 'NGA', 'students': 0, 'found_ht': 0.5, 'val_struct': 958384.635, 'val_cont': 958384.635, 'val_vehic': 81000, 'source': 'E', 'med_yr_blt': 1939, 'firmzone': 'NA', 'o65disable': 0.22, 'u65disable': 0.03, 'x': -122.2535494, 'y': 37.85648153, 'ground_elv': 239.3753899307251, 'ground_elv_m': 72.96161651611328, 'lon': -122.2539358, 'lat': 37.882252355, 'fparea': 1592, 'repaircost': 389506.045, 'constype': 'RM1', 'occupancy': 'RES1'}
53Key:  13333 Asset:
54	 Coordinates:  [[-122.2743673, 37.8721826], [-122.2743469, 37.8720391], [-122.2743897, 37.8720353], [-122.2743867, 37.8720143], [-122.2744387, 37.8720097], [-122.2744416, 37.8720305], [-122.2744612, 37.8720287], [-122.2744816, 37.8721725], [-122.2743673, 37.8721826]]
55	 Features:  {'type': 'Building', 'buildingheight': 'NA', 'erabuilt': 1992, 'numstories': 1, 'roofshape': 'NA', 'fpAreas': 1866, 'fd_id': 473584913, 'bid': '849VVPJW+WC4-6-2-5-3', 'occtype': 'RES1-1SWB', 'st_damcat': 'RES', 'bldgtype': 'M', 'found_type': 'B', 'cbfips': '060014216002007', 'pop2amu65': 2, 'pop2amo65': 1, 'pop2pmu65': 2, 'pop2pmo65': 1, 'sqft': 1592, 'num_story': 1, 'ftprntid': '06001_813170', 'ftprntsrc': 'NGA', 'students': 0, 'found_ht': 2, 'val_struct': 389506.045, 'val_cont': 194753.022, 'val_vehic': 27000, 'source': 'P', 'med_yr_blt': 1954, 'firmzone': 'NA', 'o65disable': 0.22, 'u65disable': 0.03, 'x': -122.2539358, 'y': 37.882252355, 'ground_elv': 1004.1034449096679, 'ground_elv_m': 306.05072021484375, 'lon': -122.2743974, 'lat': 37.872097621, 'fparea': 1539.29, 'repaircost': 320524.48, 'constype': 'W1', 'occupancy': 'RES3A'}
56Key:  1989 Asset:
57	 Coordinates:  [[-122.2534638, 37.856443], [-122.2534705, 37.856479], [-122.2535403, 37.8564711], [-122.253547, 37.8564912], [-122.2535946, 37.8565177], [-122.2535993, 37.8565277], [-122.2536127, 37.8565272], [-122.2535939, 37.8564404], [-122.253494, 37.8564531], [-122.2534913, 37.8564399], [-122.2534638, 37.856443]]
58	 Features:  {'type': 'Building', 'buildingheight': 'NA', 'erabuilt': 1939, 'numstories': 1, 'roofshape': 'NA', 'fpAreas': 625, 'fd_id': 474705763, 'bid': '849VVPCG+R6Q-4-2-3-2', 'occtype': 'RES3A', 'st_damcat': 'RES', 'bldgtype': 'W', 'found_type': 'S', 'cbfips': '060014223003010', 'pop2amu65': 1, 'pop2amo65': 0, 'pop2pmu65': 0, 'pop2pmo65': 0, 'sqft': 1539.29, 'num_story': 1, 'ftprntid': '06001_34778', 'ftprntsrc': 'NGA', 'students': 0, 'found_ht': 0.5, 'val_struct': 320524.48, 'val_cont': 160262.24, 'val_vehic': 27000, 'source': 'P', 'med_yr_blt': 1992, 'firmzone': 'NA', 'o65disable': 0.22, 'u65disable': 0.03, 'x': -122.2743974, 'y': 37.872097621, 'ground_elv': 165.9004509815979, 'ground_elv_m': 50.56645584106445, 'lon': -122.2535494, 'lat': 37.85648153, 'fparea': 6506.02409, 'repaircost': 958384.635, 'constype': 'W1', 'occupancy': 'IND6'}
59Key:  9361 Asset:
60	 Coordinates:  [[-122.2654248, 37.8550534], [-122.2654134, 37.854993], [-122.2654737, 37.8549859], [-122.2654851, 37.8550463], [-122.2654248, 37.8550534]]
61	 Features:  {'type': 'Building', 'buildingheight': 'NA', 'erabuilt': 'NA', 'numstories': 'NA', 'roofshape': 'NA', 'fpAreas': 392}
62Key:  17479 Asset:
63	 Coordinates:  [[-122.292788, 37.8763201], [-122.2927717, 37.8762696], [-122.2928486, 37.8762541], [-122.2928649, 37.8763047], [-122.292788, 37.8763201]]
64	 Features:  {'type': 'Building', 'buildingheight': 'NA', 'erabuilt': 'NA', 'numstories': 'NA', 'roofshape': 'NA', 'fpAreas': 435}

NSI Integration Notebook

Here is a link to a Jupyter Notebook that runs the basic code, accompanied by graphics to better illustrate the output.

Note

  1. Information on the fields output for NSI can be found here

  2. When the number of buildings in the NSI differs from the inventory it is being merged with, imputation may be required during the integration process.

  3. The NSI is new and under development and as a consequence is not perfect.