umnetdb-utils 0.1.0__tar.gz

umnetdb_utils-0.1.0/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.3
+Name: umnetdb-utils
+Version: 0.1.0
+Summary: Helper classes for querying UMnet databases
+License: MIT
+Author: Amy Liebowitz
+Author-email: amylieb@umich.edu
+Requires-Python: >=3.10
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: oracledb (>=3.1.0,<4.0.0)
+Requires-Dist: psycopg[binary] (>=3.2.9,<4.0.0)
+Requires-Dist: sqlalchemy (>=2.0.41,<3.0.0)
+Description-Content-Type: text/markdown
+
+# umnetdb-utils
+Helper classes for gathering data from umnet databases
+

umnetdb_utils-0.1.0/README.md

@@ -0,0 +1,2 @@
+# umnetdb-utils
+Helper classes for gathering data from umnet databases

umnetdb_utils-0.1.0/pyproject.toml

@@ -0,0 +1,24 @@
+[project]
+name = "umnetdb-utils"
+version = "0.1.0"
+description = "Helper classes for querying UMnet databases"
+authors = [
+    {name = "Amy Liebowitz",email = "amylieb@umich.edu"}
+]
+license = {text = "MIT"}
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "sqlalchemy (>=2.0.41,<3.0.0)",
+    "oracledb (>=3.1.0,<4.0.0)",
+    "psycopg[binary] (>=3.2.9,<4.0.0)"
+]
+
+[tool.poetry]
+
+[tool.poetry.group.dev.dependencies]
+ruff = "^0.11.10"
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"

umnetdb_utils-0.1.0/umnetdb_utils/__init__.py

@@ -0,0 +1,4 @@
+from .umnetequip import UMnetequip
+from .umnetinfo import UMnetinfo
+from .umnetdisco import Umnetdisco
+from .umnetdb import UMnetdb

umnetdb_utils-0.1.0/umnetdb_utils/base.py

@@ -0,0 +1,140 @@
+
+from os import getenv
+import re
+import logging
+
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import Session
+
+logger = logging.getLogger(__name__)
+
+class UMnetdbBase:
+    """
+    Base helper class
+    """
+    # set in child classes - you can use environment variables within curly braces here
+    URL = None
+
+    def __init__(self):
+        """
+        Initiate a umnetdb object. Note that you must provide either a url or a path
+        to an env file that looks like the 'sample_env' provided in this repo.
+        If both are provided, the url takes precedence.
+        """
+        self.url = self._resolve_url()
+        self.engine = create_engine(self.url)
+        self.session = None
+
+    def _resolve_url(self) -> str:
+        """
+        Resolves any reference to environment variables in the url attribute
+        and returns the string. The string should use curly braces to indicate an environment
+        variable
+        """
+        url = self.URL
+        for m in re.finditer(r"{(\w+)}", url):
+            var = m.group(1)
+
+            if not getenv(var):
+                raise ValueError(f"Undefined environment variable {var} in {url}")
+            url = re.sub(r"{" + var + "}", getenv(var), url)
+
+        return url
+
+    def open(self):
+        """
+        Create a new session to the database if there isn't one already
+        """
+        if not self.session:
+            self.session = Session(self.engine)
+
+    def close(self):
+        """
+        Closes db session if there is one
+        """
+        if self.session:
+            self.session.close()
+            self.session = None
+
+    def __enter__(self):
+        self.open()
+        return self
+
+    def __exit__(self, fexc_type, exc_val, exc_tb):
+        self.close()
+
+    def __getattr__(self, val:str):
+        if self.session:
+            return getattr(self.session, val)
+
+        raise AttributeError(self)
+
+    def _build_select(self, select, table, joins=None, where=None, order_by=None, limit=None, group_by=None, distinct=False):
+        '''
+        Generic 'select' query string builder built from standard query components as input.
+        The user is required to generate substrings for the more complex inputs
+        (eg joins, where statements), this function just puts all the components
+        together in the right order with appropriately-added newlines (for debugging purposes)
+        and returns the result.
+
+        :select: a list of columns to select
+          ex: ["nip.mac", "nip.ip", "n.switch", "n.port"]
+        :table: a string representing a table (or comma-separated tables, with our without aliases)
+          ex: "node_ip nip"
+        :joins: a list of strings representing join statements. Include the actual 'join' part!
+          ex: ["join node n on nip.mac = n.mac", "join device d on d.ip = n.switch"]
+        :where: A list of 'where' statements without the 'where'. The list of statements are
+          "anded". If you need "or", embed it in one of your list items
+          ex: ["node_ip.ip = '1.2.3.4'", "node.switch = '10.233.0.5'"]
+        :order_by: A string representing a column name (or names) to order by
+        :group_by: A string representing a column name (or names) to group by
+        :limit: An integer
+
+        '''
+
+        # First part of the sql statement is the 'select'
+        distinct = 'distinct ' if distinct else ''
+        sql = f"select {distinct}" + ", ".join(select) + "\n"
+
+        # Next is the table
+        sql += f"from {table}\n"
+
+        # Now are the joins. The 'join' keyword(s) need to be provided
+        # as part of the input, allowing for different join types
+        if joins:
+            for j in joins:
+                sql += f"{j}\n"
+
+        # Next are the filters. They are 'anded'
+        if where:
+            sql += "where\n"
+            sql += " and\n".join(where) + "\n"
+
+        # Finally the other options
+        if order_by:
+            sql += f"order by {order_by}\n"
+
+        if group_by:
+            sql += f"group by {group_by}\n"
+
+        if limit:
+            sql += f"limit {limit}\n"
+        
+        logger.debug(f"Generated SQL command:\n****\n{sql}\n****\n")
+
+        return sql
+
+    def _execute(self, sql, rows_as_dict=True):
+        '''
+        Generic sqlalchemy "execute this sql command and give me all the results"
+        '''
+        with self as session:
+            r = session.execute(text(sql))
+            rows = r.fetchall()
+
+        if rows and rows_as_dict:
+            return [r._mapping for r in rows]
+        elif rows:
+            return rows
+        else:
+            return []

umnetdb_utils-0.1.0/umnetdb_utils/umnetdb.py

@@ -0,0 +1,58 @@
+
+from typing import List
+
+from sqlalchemy import text
+from .base import UMnetdbBase
+
+
+class UMnetdb(UMnetdbBase):
+
+    URL="postgresql+psycopg://{UMNETDB_USER}:{UMNETDB_PASSWORD}@wintermute.umnet.umich.edu/umnetdb"
+
+    def get_device_neighbors(
+        self, device: str, known_devices_only: bool = True
+    ) -> List[dict]:
+        """
+        Gets a list of the neighbors of a particular device. If the port
+        has a parent in the LAG table that is included as well.
+        Neighbor hostname is also looked up in the device table and
+        the "source of truth" hostname is returned instead of what shows
+        up in lldp neighbor.
+
+        Setting 'known_devices_only' to true only returns neighbors that are found
+        in umnet_db's device table. Setting it to false will return all lldp neighbors.
+
+        Returns results as a list of dictionary entries keyed on column names.
+        """
+
+        if known_devices_only:
+            select = [
+                "n.port",
+                "n_d.name as remote_device",
+                "n.remote_port",
+                "l.parent",
+                "n_l.parent as remote_parent"
+                ]
+        else:
+            select = [
+                "n.port",
+                "coalesce(n_d.name, n.remote_device) as remote_device",
+                "n.remote_port",
+                "l.parent",
+                "n_l.parent as remote_parent"
+            ]
+        
+        table = "neighbor n"
+
+        joins = [
+             "left outer join device n_d on n_d.hostname=n.remote_device",
+             "left outer join lag l on l.device=n.device and l.member=n.port",
+             "left outer join lag n_l on n_l.device=n_d.name and n_l.member=n.remote_port",
+        ]
+
+        where = [f"n.device='{device}'"]
+
+        query = self._build_select(select, table, joins, where)
+        result = self.session.execute(text(query))
+
+        return [dict(zip(result.keys(), r)) for r in result]

umnetdb_utils-0.1.0/umnetdb_utils/umnetdisco.py

@@ -0,0 +1,233 @@
+
+
+from .base import UMnetdbBase
+from .utils import is_ip_address, is_mac_address
+
+
+class Umnetdisco(UMnetdbBase):
+    URL="postgresql+psycopg://{NETDISCO_DB_USER}:{NETDISCO_DB_PASSWORD}@netdisco.umnet.umich.edu:5432/netdisco"
+
+    def host_arp(self, host, start_time=None, end_time=None, limit=1, active_only=False):
+        '''
+        Does an ARP query against the netdisco db for a single host. 
+        :host: A string representing a MAC address or an IPv4 address
+        '''
+
+        # what table are we querying?
+        table = 'node_ip nip'
+        joins = ['join node n on nip.mac = n.mac']
+
+        # define select statements
+        select = ['nip.mac', 'nip.ip', 'n.switch', 'n.port', 'nip.time_first', 'nip.time_last']
+
+        # First determine if this host is a MAC address or an IP
+        where = []
+        if is_mac_address(host):
+            where.append(f"nip.mac ='{host}'")
+        elif is_ip_address(host, version=4):
+            where.append(f"nip.ip ='{host}'")
+        else:
+            raise ValueError(f"{host} is not a valid IP or mac address")
+
+        # filter for specific start/end times if specified
+        if start_time and end_time:
+            where.append(f"n.time_last between timestamp '{start_time}' and timestamp '{end_time}'")
+        elif start_time:
+            where.append(f"n.time_last > timestamp '{start_time}'")
+
+        # filter for active if defined
+        if active_only:
+            where.append("nip.active = true")
+
+        # order by last seen
+        sql = self._build_select(select, table, joins=joins, where=where, order_by="time_last", limit=limit)
+        result = self._execute(sql)
+
+        return result
+
+    def arp_count(self, prefix=None, start_time=None, end_time=None, active_only=False):
+        '''
+        Queries the host data in netdisco based on prefix and start/end time.
+        First any prefix greater than or equal to the inputted prefix is searched for
+        (in the 'device_ip' table).
+
+        Then the host table is queried.
+        '''
+        
+        # We're counting all the host IPs by subnet
+        select = ['count(distinct nip.ip)', 'dip.subnet']
+        table = 'node_ip nip'
+
+        # postgres allows us to do a join based on an IPs (type inet) membership
+        # of a subnet (type cidr)
+        joins = ['join device_ip dip on nip.ip <<= dip.subnet']
+
+        # grouping by subnet is what gives us per-subnet host counts
+        group_by = 'dip.subnet'
+
+        # append all cli filtering options
+        where = [f"dip.subnet <<= inet '{prefix}'"]
+        if start_time and end_time:
+            where.append(f"nip.time_last between timestamp '{start_time}' and timestamp '{end_time}'")
+        elif start_time:
+            where.append(f"nip.time_last > timestamp '{start_time}'")
+
+        if active_only:
+            where.append("nip.active = true")
+
+        sql = self._build_select(select, table, joins=joins, where=where, group_by=group_by)
+        return self._execute(sql)
+
+    def neighbors(self, device=None):
+        '''
+        Queries the device_ip table in netdisco to get neighbors of a device.
+        If no device is specified, all neighbors are pulled.
+        The device input can be an IPv4 address or an FQDN.
+        '''
+
+        select = [ 'dp.ip as local_ip',
+                   'ld.dns as local_dns',
+                   'dp.port as local_port',
+                   'dp.remote_ip',
+                   'rd.dns as remote_dns',
+                   'dp.remote_port']
+        table = 'device d'
+        joins = ['join device_port dp on d.ip = dp.ip',
+                 'join device ld on dp.ip = ld.ip',
+                 'join device rd on dp.remote_ip = rd.ip']
+
+        where = ['dp.remote_ip is not null']
+        if is_ip_address(device):
+            where.append(f"d.ip = '{device}'")
+        elif device:
+            where.append(f"d.dns = '{device}'")
+
+        sql = self._build_select(select, table, joins, where)
+        return self._execute(sql)
+
+    def get_devices(self, match_subnets=None, exclude_subnets=None):
+        '''
+        Queries netdisco for a list of devices.
+        Optionally, limit it by a list of prefixes
+        '''
+
+        select = [ 'ip', 'dns', 'serial', 'model', 'vendor', 'os' ]
+        table = 'device'
+
+        where = []
+        if match_subnets:
+            where.append(" or\n".join([f"ip << inet '{s}'" for s in match_subnets]))
+        
+        if exclude_subnets:
+            where.append("not\n" + " or\n".join([f"ip << inet '{s}'" for s in exclude_subnets]))
+
+        sql = self._build_select(select, table, where=where)
+        return self._execute(sql)
+
+    def get_device(self, name_or_ip):
+        '''
+        Pulls the device name, ip, model, and description (where version info often is)
+        from the netinfo device table
+        '''
+        select = [
+                "replace(lower(name), '.umnet.umich.edu', '') as name",
+                "device.ip as ip",
+                "model",
+                "description",
+                ]
+        table = "device"
+        
+        if is_ip_address(name_or_ip):
+            joins = ["join device_ip on device.ip=device_ip.alias"]
+            where = [f"device_ip.alias='{name_or_ip}'"]
+        else:
+            joins = []
+            where = [f"lower(name) like '{name_or_ip.lower()}%'"]
+
+        sql = self._build_select(select, table, where=where, joins=joins, limit=1)
+        result = self._execute(sql)
+
+        if result:
+            return result[0]
+        else:
+            return None
+
+    def get_port_and_host_data(self, name_or_ip):
+        '''
+        Pulls a list of all ports on the switch and all hosts seen on those ports.
+        You will see multiple rows for an interface where multiple hosts have been seen.
+
+        Returns a list of ports along with:
+        - port name
+        - port description
+        - admin status
+        - oper status
+        - description
+        - host mac
+        - host IP
+        - vlan the host was seen on
+        - date last mac was seen
+
+        '''
+
+        # getting device by name or IP
+        if is_ip_address(name_or_ip):
+            device_ip = name_or_ip
+        else:
+            nd_device = self.get_device(name_or_ip)
+            if not(nd_device):
+                raise LookupError(f'Could not find {nd_device} in Netdisco')
+            device_ip = nd_device['ip']
+
+        # The port description is in different fields depending on the platform, so we need to do
+        # some if-else stuff that's not supported in our basic sql builer in the base class.
+        # That's why we're just doing a long sting here
+        sql = f'''
+select
+dp.port,
+case
+ when dp.name=dp.port then dp.descr
+ else dp.name
+end
+as description,
+REPLACE(dp.up,'lowerLayerDown','down') as oper_status,
+dp.up_admin as admin_status,
+dp.speed,
+node.vlan,
+dp.remote_ip,
+node.mac,
+node.time_last,
+node_ip.ip
+from device_port dp
+left outer join node on dp.port = node.port and dp.ip = node.switch
+left outer join node_ip on node.mac = node_ip.mac
+where dp.ip='{device_ip}'
+'''
+
+        result = self._execute(sql)
+
+        return result
+
+    def get_poe_data(self, name_or_ip):
+        """
+        Pulls PoE admin, status, class, and power columns from device_port_pwer 
+        """
+
+        # getting device by name or IP
+        if is_ip_address(name_or_ip):
+            device_ip = name_or_ip
+        else:
+            nd_device = self.get_device(name_or_ip)
+            if not(nd_device):
+                raise LookupError(f'Could not find {nd_device} in Netdisco')
+            device_ip = nd_device['ip']
+    
+        select = ["port", "admin", "status", "class", "power"]
+        table = "device_port_power"
+        where = [f"ip='{device_ip}'"]
+
+
+        sql = self._build_select(select, table, where=where)
+        result = self._execute(sql)
+
+        return result

umnetdb_utils-0.1.0/umnetdb_utils/umnetequip.py

@@ -0,0 +1,157 @@
+from typing import Union, Optional
+
+from .base import UMnetdbBase
+from .utils import is_ip_address
+
+class UMnetequip(UMnetdbBase):
+    '''
+    This class wraps helpful equip db queries in python.
+    '''
+    URL = "oracle+oracledb://{EQUIP_DB_USERNAME}:{EQUIP_DB_PASSWORD}@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=equipdb-prod.umnet.umich.edu)(PORT=1521))(CONNECT_DATA=(SID=KANNADA)))"
+
+    def get_devices_by_category(self, category, active_only=False):
+        '''
+        Queries equip db for devices by category. You can also
+        specify if you only want active devices.
+        '''
+
+        select = [ 'eq.monitored_device',
+                   'eq.rancid',
+                   'eq.off_line',
+                   'eq.dns_name',
+                   'eq.ip_address',
+                 ]
+        table = 'ARADMIN1.UMNET_EQUIPMENT eq'
+
+        where = [f"eq.category = '{category}'"]
+
+        # Equipshim status numbers (see ARADMIN1.UMNET_EQUIPMENT_STATUS)
+        # 1: RESERVE, 2:ACTIVE, 3:RETIRED
+        if active_only:
+            where.append("eq.status = 2")
+        
+        sql = self._build_select(select, table, where=where, distinct=True)
+        return self._execute(sql)
+
+
+    def get_device_type(self, ip):
+        '''
+        Queries equip db for a device by ip, returns the 'type' of the device.
+        By type we mean the UMNET_EQUIPMENT_TYPE: ACCESS LAYER, DISTRIBUTION LAYER, UMD, etc
+        '''
+        select = [ 'types as type' ]
+        table = 'ARADMIN1.UMNET_EQUIPMENT'
+        where = [f"ip_address='{ip}'"]
+
+        sql = self._build_select(select, table, where=where)
+        return self._execute(sql)
+
+    def get_devices_by_building_no(self, 
+                    building_no:Union[int, list],
+
+
+                    active_only:bool=False,
+                    types:Optional[list]=None,
+                    location_info:bool=False,
+                    ):
+        """
+        Queries equipdb for devices by building no. You can provide a single
+        building number, or a list of numbers. You can specify if you want 'active only' devices
+        (based on the 'status' field, defalut false) or you can limit to a certain device type (default all).
+        You can also get the location info for each device via 'location_info' (default false)
+        """
+        select = [ 'dns_name',
+                   'ip_address',
+                   'model_no_ as model',
+                   'types as device_type', 
+                   'rancid',
+                   'off_line',         
+                 ]
+        if location_info:
+            select.extend([
+                    'bldg as building_name',
+                    'address as building_address',
+                    'room as room_no',
+                    'floor as floor',    
+                    'bldg_code__ as building_no',     
+            ])
+
+        table = 'ARADMIN1.UMNET_EQUIPMENT eq'
+
+        where = []
+        if isinstance(building_no, int):
+            where.append(f"eq.bldg_code__ = {building_no}")
+        elif isinstance(building_no, list):
+            bldgs_str = ",".join([str(bldg) for bldg in building_no])
+            where.append(f"eq.bldg_code__ in ({bldgs_str})")
+        else:
+            raise ValueError(f"{building_no} must be int or list!")
+
+        # Equipshim status numbers (see ARADMIN1.UMNET_EQUIPMENT_STATUS)
+        # 1: RESERVE, 2:ACTIVE, 3:RETIRED
+        if active_only:
+            where.append("eq.status = 2")
+
+        if types:
+            types_str = ",".join([f"'{t}'" for t in types])
+            where.append(f"eq.types in ({types_str})")
+
+        sql = self._build_select(select, table, where=where)
+        return self._execute(sql)
+
+    def get_device(self, name_or_ip:str, location_info:bool=False):
+
+        if is_ip_address(name_or_ip):
+            where = [f"ip_address='{name_or_ip}'"]
+        else:
+            name_or_ip = name_or_ip.replace(".umnet.umich.edu","")
+            where = [f"dns_name='{name_or_ip}'"]
+
+        select = [ 'dns_name',
+                   'ip_address',
+                   'model_no_ as model',
+                   'types as device_type', 
+                   'rancid',
+                   'off_line',          
+                 ]
+        if location_info:
+            select.extend([
+                    'bldg as building_name',
+                    'address as building_address',
+                    'room as room_no',
+                    'floor as floor',    
+                    'bldg_code__ as building_no',     
+            ])
+        table = 'ARADMIN1.UMNET_EQUIPMENT eq'
+
+        sql = self._build_select(select, table, where=where)
+        return self._execute(sql)
+
+    def get_all_devices(self):
+        select = [  "bldg_code__ as bldg_code",
+                    "bldg",
+                    "room",
+                    "ip_address",
+                    "dns_name",
+                    "category",
+                    "types",
+                    "manufacturer",
+                    "serial_number",
+                    "billing_code_ as billing_code",
+                    "warehouse_item__ as warehouse_item",
+                    "st.descr as status",
+                    "sla_network",
+                    "address",
+                    "floor",
+                    "model_no_ as model_no",
+                    "customer_name",
+                    "mat_l_item_description",
+                    "rancid",
+                    "off_line",
+                 ]
+        table = 'ARADMIN1.UMNET_EQUIPMENT eq'
+        joins = ["join ARADMIN1.UMNET_EQUIPMENT_STATUS st on st.idnum=eq.status"]
+        where = ["eq.status != 3"]
+
+        sql = self._build_select(select, table,where=where, joins=joins)
+        return self._execute(sql)

umnetdb_utils-0.1.0/umnetdb_utils/umnetinfo.py

@@ -0,0 +1,489 @@
+
+
+import ipaddress
+import re
+from .base import UMnetdbBase
+from .utils import is_ip_address, is_ip_network, is_mac_address
+
+class UMnetinfo(UMnetdbBase):
+    """
+    Wraps helpful netinfo db queries into python
+    """
+    URL = "oracle+oracledb://{NETINFO_USERNAME}:{NETINFO_PASSWORD}@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=kannada.web.itd.umich.edu)(PORT=1521))(CONNECT_DATA=(SID=KANNADA)))"
+
+    def get_network_by_name(self, netname, active_only=False):
+        """
+        Looks up a network by netname.
+        """
+        select = [
+            "p.itemname as netname",
+            "i.itemname as ipv4_subnet",
+            "six_i.itemname as ipv6_subnet",
+            "NVL(n.vlanid,n.local_vlanid) as vlan_id",
+            "s.statusdes as status",
+        ]
+        table = "UMNET_ONLINE.ITEM p"
+        joins = [
+            "join UMNET_ONLINE.NETWORK n on p.itemidnum = n.itemidnum",
+            "join UMNET_ONLINE.STATUS_CODE s on p.statuscd = s.statuscd",
+            "left outer join UMNET_ONLINE.ITEM i on i.parentitemidnum = p.itemidnum",
+            "left outer join UMNET_ONLINE.IP_SUBNET ip on i.itemidnum = ip.itemidnum",
+            "left outer join UMNET_ONLINE.RELATED_ITEM r on p.itemidnum = r.relitemidnum",
+            "left outer join UMNET_ONLINE.IP6NET six on r.itemidnum = six.itemidnum",
+            "left outer join UMNET_ONLINE.ITEM six_i on six.itemidnum = six_i.itemidnum",
+        ]
+        where = [
+            f"p.itemname = '{netname}'",
+        ]
+
+        sql = self._build_select(select, table, joins=joins, where=where)
+        results = self._execute(sql)
+
+        # there are also IPv4 subnets that are "pending removal" or "pending activation"
+        # tied to the network in the RELATED_ITEM table, we need to find those too.
+        # doing it as a separate query to maintain existing results row structure
+        select = [
+            "p.itemname as netname",
+            "r_i.itemname as ipv4_subnet",
+            "'' as ipv6_subnet",
+            "NVL(n.vlanid,n.local_vlanid) as vlan_id",
+            "r.itemreltypcd as status",
+        ]
+        joins = [
+            "join UMNET_ONLINE.NETWORK n on p.itemidnum = n.itemidnum",
+            "left outer join UMNET_ONLINE.ITEM i on i.parentitemidnum = p.itemidnum",
+            "left outer join UMNET_ONLINE.RELATED_ITEM r on p.itemidnum = r.relitemidnum",
+            "left outer join UMNET_ONLINE.ITEM r_i on r.itemidnum = r_i.itemidnum",
+        ]
+        where = [
+            f"p.itemname = '{netname}'",
+            "r_i.itemcatcd = 'IP'",
+        ]
+        # if we're doing active only, we only want "pending removal"
+        if active_only:
+            where.append("r.itemreltypcd = 'IP-PRNET'")
+
+        sql = self._build_select(select, table, joins=joins, where=where)
+        more_results = self._execute(sql)
+
+        if more_results:
+            results.extend(more_results)
+
+        return results
+
+    def get_network_by_ip(self, ip, active_only=False):
+        """
+        Looks up a network based on an IPv4 or IPv6 address. Returns the netname,
+        vlan id, as well as *all active subnets* (IPv4 and IPv6) tied to the netname.
+        """
+
+        ip = ipaddress.ip_address(ip)
+
+        # to make our lives simpler we're breaking this up into two steps.
+        # first let's find the network entry in the 'item' table
+        if ip.version == 4:
+            select = ["NVL(p.itemidnum, r.relitemidnum) as id"]
+            table = "UMNET_ONLINE.IP_SUBNET ip"
+            joins = [
+                "join UMNET_ONLINE.ITEM i on ip.itemidnum = i.itemidnum",
+                "left outer join UMNET_ONLINE.RELATED_ITEM r on r.itemidnum = ip.itemidnum",
+                "left outer join UMNET_ONLINE.ITEM p on i.parentitemidnum = p.itemidnum",
+            ]
+            where = [
+                f"{int(ip)} >= ip.ADDRESS32BIT",
+                f"{int(ip)} <= ip.ENDADDRESS32BIT",
+                "NVL(p.itemidnum, r.relitemidnum) is not null",
+            ]
+
+        # IPv6 table is related to the 'item' table via the RELATED_ITEM table
+        elif ip.version == 6:
+            select = ["p.itemidnum as id"]
+            table = "UMNET_ONLINE.IP6NET ip"
+            joins = [
+                "join UMNET_ONLINE.RELATED_iTEM r on r.itemidnum = ip.itemidnum",
+                "join UMNET_ONLINE.ITEM p on p.itemidnum = r.relitemidnum",
+            ]
+            # and the start/end addresses are stored as hex strings
+            addr_str = ip.exploded.replace(":", "")
+            where = [
+                f"'{addr_str}' >= ip.ADDRESS128BIT",
+                f"'{addr_str}' <= ip.ENDADDRESS128BIT",
+            ]
+
+        sql = self._build_select(select, table, joins=joins, where=where)
+        network = self._execute(sql)
+
+        if not (network):
+            return False
+        net_id = network[0]["id"]
+
+        # Now let's use the network itemidnum to find all associated subnets
+        # with that network, as well as the netname and VLAN ID
+        select = [
+            "p.itemname as netname",
+            "i.itemname as ipv4_subnet",
+            "six_i.itemname as ipv6_subnet",
+            "NVL(n.vlanid,n.local_vlanid) AS vlan_id",
+            "i.itemcatcd as ITEMCAT",
+            "p.itemcatcd as PCAT",
+            "r.itemreltypcd as RELCAT",
+        ]
+        table = "UMNET_ONLINE.ITEM p"
+        joins = [
+            "join UMNET_ONLINE.NETWORK n on p.itemidnum = n.itemidnum",
+            "join UMNET_ONLINE.ITEM i on i.parentitemidnum = p.itemidnum",
+            "left outer join UMNET_ONLINE.RELATED_ITEM r on p.itemidnum = r.relitemidnum",
+            "left outer join UMNET_ONLINE.IP6NET six on r.itemidnum = six.itemidnum",
+            "left outer join UMNET_ONLINE.ITEM six_i on six.itemidnum = six_i.itemidnum",
+        ]
+        where = [
+            f"p.itemidnum = {net_id}",
+        ]
+
+        sql = self._build_select(select, table, joins=joins, where=where)
+        results = self._execute(sql)
+
+        # there are also IPv4 subnets that are "pending removal" or "pending activation"
+        # tied to the network in the RELATED_ITEM table, we need to find those too.
+        # doing it as a separate query to maintain existing results row structure
+        select = [
+            "p.itemname as netname",
+            "r_i.itemname as ipv4_subnet",
+            "'' as ipv6_subnet",
+            "NVL(n.vlanid,n.local_vlanid) AS vlan_id",
+            "r.itemreltypcd as RELTYPCD",
+        ]
+        joins = [
+            "join UMNET_ONLINE.NETWORK n on p.itemidnum = n.itemidnum",
+            "left outer join UMNET_ONLINE.ITEM i on i.parentitemidnum = p.itemidnum",
+            "left outer join UMNET_ONLINE.RELATED_ITEM r on p.itemidnum = r.relitemidnum",
+            "left outer join UMNET_ONLINE.ITEM r_i on r.itemidnum = r_i.itemidnum",
+        ]
+        where = [
+            f"p.itemidnum = {net_id}",
+            "r_i.itemcatcd = 'IP'",
+        ]
+        # if we're doing active only, we only want "pending removal"
+        if active_only:
+            where.append("r.itemreltypcd = 'IP-PRNET'")
+
+        sql = self._build_select(select, table, joins=joins, where=where)
+        more_results = self._execute(sql)
+
+        if more_results:
+            results.extend(more_results)
+
+        return results
+
+    def get_vrfs(self, vrf_name=None, rd=None):
+        """
+        Pulls data from the vrf table on netinfo. If you supply a name and/or rd, it filters only
+        for that vrf. Otherwise it returns all VRFs
+        """
+
+        select = [
+            "shortname",
+            "route_distinguisher",
+            "default_vrf",
+            "inside_vrf",
+        ]
+        table = "UMNET_ONLINE.VRF"
+
+        where = []
+        if vrf_name:
+            where.append(f"shortname = '{vrf_name}'")
+        if rd:
+            where.append(f"route_distinguisher = '{rd}'")
+
+        sql = self._build_select(select, table, where=where)
+        results = self._execute(sql)
+
+        return results
+
+    def get_special_acl(self, netname: str):
+        """
+        Looks for a special ACL assignment by netname
+        """
+        select = ["acl.itemname"]
+        table = "UMNET_ONLINE.ITEM net"
+        joins = [
+            "join UMNET_ONLINE.FILTER_NETWORK fn on fn.net_itemidnum = net.itemidnum",
+            "join UMNET_ONLINE.ITEM acl on fn.filter_itemidnum = acl.itemidnum",
+        ]
+        where = [f"net.itemname='{netname}'"]
+
+        sql = self._build_select(select, table, joins=joins, where=where)
+        results = self._execute(sql)
+
+        return results
+
+    def get_asns(self, name_filter: str = ""):
+        """
+        Pulls all the ASNs from the AUTONOMOUS_SYSTEM
+        table, optionally filtering by asname
+        """
+        select = ["ASNAME", "ASN"]
+        table = "UMNET_ONLINE.AUTONOMOUS_SYSTEM"
+
+        where = []
+        if name_filter:
+            where = [f"ASNAME like '%{name_filter}%'"]
+
+        sql = self._build_select(select, table, where=where)
+        results = self._execute(sql)
+
+        return results
+
+    def get_dlzone_buildings(self, zone: str):
+        """
+        given a dlzone shortname, returns a list of building numbers tied
+        to that zone.
+        """
+
+        select = ["BUILDINGNUM as building_no"]
+        table = "UMNET_ONLINE.AUTONOMOUS_SYSTEM asn"
+        joins = [
+            "join UMNET_ONLINE.BUILDING_AS b_asn on b_asn.ASID = asn.ASID",
+        ]
+        where = [f"asn.ASNAME = '{zone}'"]
+
+        sql = self._build_select(select, table, joins=joins, where=where)
+        results = self._execute(sql)
+
+        return results
+
+    def get_dlzone_by_building(self, bldg_num: int):
+        """
+        given a seven-digit building number, return the name of the
+        distribution-zone to which it is assigned
+        """
+
+        select = ["ASNAME"]
+        table = "UMNET_ONLINE.AUTONOMOUS_SYSTEM asn"
+        joins = [
+            "join UMNET_ONLINE.BUILDING_AS b_asn on b_asn.ASID = asn.ASID",
+        ]
+        where = [f"b_asn.BUILDINGNUM = '{bldg_num}'"]
+
+        sql = self._build_select(select, table, joins=joins, where=where)
+        results = self._execute(sql)
+
+        return results
+
+    def get_user_groups(self, username: str):
+        """
+        Looks up a user in netinfo and returns the groups they're in
+        """
+
+        select = ["e_grp.name"]
+        table = "UMNET_ONLINE.PERSON p"
+        joins = [
+            "join UMNET_ONLINE.PERSON_GROUP p_grp on p_grp.PERSON_ENTITYIDNUM = p.ENTITYIDNUM",
+            "join UMNET_ONLINE.ENTITY_GROUP e_grp on e_grp.ENTITYIDNUM = p_grp.GROUP_ENTITYIDNUM",
+        ]
+        where = [f"UNIQNAME='{username}'"]
+
+        sql = self._build_select(select, table, joins=joins, where=where)
+        results = self._execute(sql)
+
+        return results
+
+    def get_network_admins(
+        self, netname: str, entity_types: list = [], rel_types: list = []
+    ):
+        """
+        Looks up the users and user groups that are tied to this network.
+        You can fiter by entity types (see UMNET_ONLINE.ENTITY_TYPE_CODE), these are:
+          group, organization, person
+        You can also filter by relationship type (UMNET_ONLINE.ENTITEM_RELATIONSHIP_TYPE_CODE):
+          Owner, Worker, Business, Admin, Security
+        """
+
+        select = [
+            "NVL(NVL(e_g.name, p.uniqname), o.NAME) as name",
+            "e_t_c.ENTITYTYPDES as entity_type",
+            "i_a.ENTITYITEMRELTYPCD as rel_type",
+        ]
+
+        table = "UMNET_ONLINE.ITEM i"
+        joins = [
+            "join UMNET_ONLINE.ITEM_ADMIN i_a on i_a.ITEMIDNUM=i.ITEMIDNUM",
+            "join UMNET_ONLINE.ENTITY e on i_a.ENTITYIDNUM=e.ENTITYIDNUM",
+            "join UMNET_ONLINE.ENTITY_TYPE_CODE e_t_c on e_t_c.ENTITYTYPCD=e.ENTITYTYPCD",
+            "left outer join UMNET_ONLINE.ENTITY_GROUP e_g on e_g.ENTITYIDNUM=e.ENTITYIDNUM",
+            "left outer join UMNET_ONLINE.PERSON p on p.ENTITYIDNUM=e.ENTITYIDNUM",
+            "left outer join UMNET_ONLINE.ORGANIZATION o on o.ENTITYIDNUM=e.ENTITYIDNUM",
+        ]
+
+        where = [f"i.ITEMNAME='{netname}'"]
+
+        if entity_types:
+            e_where = " or ".join([f"e_t_c.ENTITYTYPDES='{e}'" for e in entity_types])
+            where.append(f"({e_where})")
+        if rel_types:
+            rel_where = " or ".join(
+                [f"i_a.ENTITYITEMRELTYPCD='{rel}'" for rel in rel_types]
+            )
+            where.append(f"({rel_where})")
+
+        sql = self._build_select(select, table, joins=joins, where=where)
+        results = self._execute(sql)
+
+        return results
+
+    def get_device_admins(
+        self, name_or_ip: str, entity_types: list = [], rel_types: list = []
+    ):
+        """
+        Looks up the users and user groups that are tied to this device.
+        You can fiter by entity types (see UMNET_ONLINE.ENTITY_TYPE_CODE), these are:
+          group, organization, person
+        You can also filter by relationship type (UMNET_ONLINE.ENTITEM_RELATIONSHIP_TYPE_CODE):
+          Owner, Worker, Business, Admin, Security
+        """
+
+        select = [
+            "NVL(NVL(e_g.name, p.uniqname), o.NAME) as name",
+            "e_t_c.ENTITYTYPDES as entity_type",
+            "i_a.ENTITYITEMRELTYPCD as rel_type",
+        ]
+
+        table = "UMNET_ONLINE.DEVICE d"
+        joins = [
+            "join UMNET_ONLINE.ITEM i on i.itemidnum=d.ITEMIDNUM",
+            "join UMNET_ONLINE.ITEM_ADMIN i_a on i_a.ITEMIDNUM=i.ITEMIDNUM",
+            "join UMNET_ONLINE.ENTITY e on i_a.ENTITYIDNUM=e.ENTITYIDNUM",
+            "join UMNET_ONLINE.ENTITY_TYPE_CODE e_t_c on e_t_c.ENTITYTYPCD=e.ENTITYTYPCD",
+            "left outer join UMNET_ONLINE.ENTITY_GROUP e_g on e_g.ENTITYIDNUM=e.ENTITYIDNUM",
+            "left outer join UMNET_ONLINE.PERSON p on p.ENTITYIDNUM=e.ENTITYIDNUM",
+            "left outer join UMNET_ONLINE.ORGANIZATION o on o.ENTITYIDNUM=e.ENTITYIDNUM",
+        ]
+
+        # netinfo IPv4 addresses are always stored as integers
+        if is_ip_address(name_or_ip):
+            where = [f"d.ADDRESS32={int(ipaddress.ip_address(name_or_ip))}"]
+
+        # dns names in the device table are fqdn and end in a dot (eg 'dl-arbl-1.umnet.umich.edu.')
+        else:
+
+            name_or_ip = name_or_ip.lower()
+            if "." not in name_or_ip:
+                name_or_ip += ".umnet.umich.edu"
+            if not (name_or_ip.endswith(".")):
+                name_or_ip += "."
+
+            where = [f"d.DNS_NAME='{name_or_ip}'"]
+
+        if entity_types:
+            e_where = " or ".join([f"e_t_c.ENTITYTYPDES='{e}'" for e in entity_types])
+            where.append(f"({e_where})")
+        if rel_types:
+            rel_where = " or ".join(
+                [f"i_a.ENTITYITEMRELTYPCD='{rel}'" for rel in rel_types]
+            )
+            where.append(f"({rel_where})")
+
+        sql = self._build_select(select, table, joins=joins, where=where)
+        results = self._execute(sql)
+
+        return results
+
+    def get_arphist(
+        self, query: str, device=None, interface=None, no_hist=False, no_device=False
+    ):
+        """
+        Given either a MAC address, IP address, or subnet, query UMNET.ARPHIST
+        and return the results.
+
+        If you don't care about history, set "no_hist"
+        If you don't care about which device the entries are routed on,
+        set "no_device"
+
+        Optionally limit the query by device and/or by interface
+        with the "device" and "interface" fields
+        """
+
+        # for 'no_hist' 'no_device' we are just straight up pulling ip->mac
+        # mappings
+        select = ["address32bit, mac_addr"]
+        if not no_hist:
+            select.extend(["first_seen", "last_seen"])
+        if not no_device:
+            select.extend(["device_name as device", "ifdescr as interface"])
+
+        table = "UMNET.ARPHIST arp"
+        where = []
+
+        # UMNET.ARPHIST stores MACs as strings without separators .:-, ie
+        # 0010.abcd.1010 => '0010abcd1010'
+        if is_mac_address(query):
+
+            arphist_mac = re.sub(r"[\.\:\-]", "", query)
+            where.append(f"arp.mac_addr = '{arphist_mac}'")
+
+        # UMNET.ARPHIST stores IPs as integers, ie
+        # 10.233.0.10 => 183042058. Also - only supports IPv4
+        # IPv6 is stored in IP6NEIGHBOR
+        elif is_ip_address(query, version=4):
+            ip = ipaddress.ip_address(query)
+            where.append(f"arp.address32bit = {int(ip)}")
+
+        elif is_ip_network(query, version=4):
+            net = ipaddress.ip_network(query)
+            where.extend(
+                [
+                    f"arp.address32bit >= {int(net[0])}",
+                    f"arp.address32bit <= {int(net[-1])}",
+                ]
+            )
+
+        else:
+            raise ValueError(
+                f"Unsupported input {query}, must be a MAC, a subnet, or an IP"
+            )
+
+        if device:
+            where.append(f"arp.device = '{device}'")
+        if interface:
+            where.append(f"arp.interface = '{interface}'")
+
+        sql = self._build_select(select, table, where=where, distinct=True)
+        results = self._execute(sql)
+
+        # normally we like to return the results unprocessed. But the IPs and
+        # MACs really need to get converted to make the ouptut useful.
+        # hopefully the type different (Sqlalchemy result object vs list[dict])
+        # isn't throwing anyone :-/
+        processed_results = []
+        for r in results:
+
+            processed = {
+                "ip": ipaddress.ip_address(r["address32bit"]),
+                "mac": f"{r['mac_addr'][0:4]}.{r['mac_addr'][4:8]}.{r['mac_addr'][8:12]}",
+            }
+            for col in ["device", "interface", "first_seen", "last_seen"]:
+                if col in r:
+                    processed[col] = r[col]
+
+            processed_results.append(processed)
+
+        return processed_results
+
+    def get_prefix_lists(self, prefix_lists: list = None):
+        """
+        Queries UMNET_ONLINE.PREFIX_LISTS for a list of prefix list names (as defined in the database).
+        If you don't provide a list of names it will return all of them
+        """
+
+        select = ["name", "prefix"]
+        table = "UMNET_ONLINE.PREFIX_LIST"
+
+        where = []
+        if prefix_lists:
+            in_query = "(" + ",".join([f"'{p}'" for p in prefix_lists]) + ")"
+            where.append(f"name in {in_query}")
+
+        sql = self._build_select(select, table, where=where, order_by="1,2")
+        results = self._execute(sql)
+
+        return results

umnetdb_utils-0.1.0/umnetdb_utils/utils.py

@@ -0,0 +1,39 @@
+import ipaddress
+import re
+def is_ip_address(input_str, version=None):
+    try:
+        ip = ipaddress.ip_address(input_str)
+    except ValueError:
+        return False
+
+    if version and version != ip.version:
+        return False
+
+    return True
+
+def is_ip_network(input_str, version=None):
+ 
+    # First check that this is a valid IP or network
+    try:
+        net = ipaddress.ip_network(input_str)
+    except ValueError:
+        return False
+
+    if version and version != net.version:
+        return False
+    
+    return True
+
+def is_mac_address(input_str):
+    '''
+    Validates the input string as a mac address. Valid formats are
+    XX:XX:XX:XX:XX:XX, XX-XX-XX-XX-XX-XX, XXXX.XXXX.XXXX
+    where 'X' is a hexadecimal digit (upper or lowercase).
+    '''
+    mac = input_str.lower()
+    if re.match(r'[0-9a-f]{2}([-:])[0-9a-f]{2}(\1[0-9a-f]{2}){4}$', mac):
+        return True
+    if re.match(r'[0-9a-f]{4}\.[0-9a-f]{4}\.[0-9a-f]{4}$', mac):
+        return True
+
+    return False