Merge branch 'master' into device-roles

README.md

@@ -32,24 +32,71 @@ The goal is to generate an existing infrastructure on Netbox and have the abilit
 - lldpd
 - lshw
 
-# Known limitations
+## Inventory requirement
+- hpassacli
+- storcli
+- omreport
 
-* The project is only compatible with Linux.
+# Installation
-Since it uses `ethtool` and parses `/sys/` directory, it's not compatible with *BSD distributions.
+
-* Netbox `>=2.6.0,<=2.6.2` has a caching problem ; if the cache lifetime is too high, the script can get stale data after modification.
+```
-We advise to set `CACHE_TIME` to `0`.
+# pip3 install netbox-agent
+```
+
+# Usage
+
+The agent can be run from a shell and get its configuration from either the configuration file or environment variables.
+
+Configuration values are overridden based on the following precedence: command line arguments (might include config file) > environment variables > default config file > defaults.
+
+```
+# netbox_agent -c /etc/netbox_agent.yml --register
+INFO:root:Creating chassis blade (serial: QTFCQ574502EF)
+INFO:root:Creating blade (serial: QTFCQ574502D2) myserver on chassis QTFCQ574502EF
+INFO:root:Setting device (QTFCQ574502D2) new slot on Slot 9 (Chassis QTFCQ574502EF)..
+INFO:root:Interface a8:1e:84:f2:9e:6a not found, creating..
+INFO:root:Creating NIC enp1s0f1 (a8:1e:84:f2:9e:6a) on myserver
+INFO:root:Interface 02:42:7a:89:cf:a4 not found, creating..
+INFO:root:Creating NIC br-07ea1e4a2f0e (02:42:7a:89:cf:a4) on myserver
+INFO:root:Create new IP 172.19.0.1/16 on br-07ea1e4a2f0e
+INFO:root:Interface a8:1e:84:f2:9e:69 not found, creating..
+INFO:root:Creating NIC enp1s0f0 (a8:1e:84:f2:9e:69) on myserver
+INFO:root:Create new IP 42.42.42.42/24 on enp1s0f0
+INFO:root:Create new IP fe80::aa1e:84ff:fef2:9e69/64 on enp1s0f0
+INFO:root:Interface a8:1e:84:cd:9d:d6 not found, creating..
+INFO:root:Creating NIC IPMI (a8:1e:84:cd:9d:d6) on myserver
+INFO:root:Create new IP 10.191.122.10/24 on IPMI
+```
+
+If you need, you can update only specific informations like:
+* Network
+* Inventory
+* Location
+* PSUs
+
+```
+# ip a add 42.42.42.43/24 dev enp1s0f1
+# netbox_agent -c /etc/netbox_agent.yaml --update-network
+INFO:root:Create new IP 42.42.42.43/24 on enp1s0f1
+# netbox_agent --update-inventory
+INFO:root:Creating Disk Samsung SSD 850 S2RBNX0K101698D
+```
 
 # Configuration
 
 ```
+# Netbox configuration
 netbox:
  url: 'http://netbox.internal.company.com'
  token: supersecrettoken
 
+# Network configuration
 network:
+  # Regex to ignore interfaces 
   ignore_interfaces: "(dummy.*|docker.*)"
+  # Regex to ignore IP addresses
   ignore_ips: (127\.0\.0\..*)
-  # enable auto-cabling
+  # enable auto-cabling by parsing LLDP answers
   lldp: true
 
 #
@@ -69,12 +116,14 @@ network:
 # driver: "file:/tmp/tenant"
 # regex: "(.*)"
 
+## Enable virtual machine support 
 # virtual:
 #   # not mandatory, can be guessed
 #   enabled: True
 #   # see https://netbox.company.com/virtualization/clusters/
 #   cluster_name: my_vm_cluster
 
+# Enable datacenter location feature in Netbox
 datacenter_location:
  driver: "cmd:cat /etc/qualification | tr [a-z] [A-Z]"
  regex: "DATACENTER: (?P<datacenter>[A-Za-z0-9]+)"
@@ -84,6 +133,7 @@ datacenter_location:
 # driver: "file:/tmp/datacenter"
 # regex: "(.*)"
 
+# Enable rack location feature in Netbox
 rack_location:
 # driver: 'cmd:lldpctl'
 # match SysName: sw-dist-a1.dc42
@@ -92,6 +142,7 @@ rack_location:
 # driver: "file:/tmp/datacenter"
 # regex: "(.*)"
 
+# Enable local inventory reporting 
 inventory: true
 ```
 
@@ -178,6 +229,27 @@ Feel free to send me a dmidecode output for Supermicro's blade!
 
 * Nothing ATM, feel free to send me a dmidecode or make a PR!
 
-# TODO
+# Known limitations
 
-- [ ] `CustomFields` support with firmware versions for Device (BIOS), RAID Cards and disks
+* The project is only compatible with Linux.
+Since it uses `ethtool` and parses `/sys/` directory, it's not compatible with *BSD distributions.
+* Netbox `>=2.6.0,<=2.6.2` has a caching problem ; if the cache lifetime is too high, the script can get stale data after modification.
+We advise to set `CACHE_TIME` to `0`.
+
+# Developing
+
+If you want to run the agent while adding features or just for debugging purposes
+
+```
+# git clone https://github.com/Solvik/netbox-agent.git
+# cd netbox-agent
+# python3 -m netbox_agent.cli --register
+```
+
+On a personal note, I use the docker image from [netbox-community/netbox-docker](https://github.com/netbox-community/netbox-docker)
+```
+# git clone https://github.com/netbox-community/netbox-docker
+# cd netbox-docker
+# docker-compose pull
+# docker-compose up 
+```

netbox_agent/inventory.py

@@ -139,7 +139,7 @@ class Inventory():
 
         for nb_motherboard in nb_motherboards:
             if nb_motherboard.serial not in [x['serial'] for x in motherboards]:
-                logging.info('Deleting unknown motherboard {vendor} {motherboard}/{serial}'.format(
+                logging.info('Deleting unknown motherboard {motherboard}/{serial}'.format(
                     motherboard=self.lshw.motherboard,
                     serial=nb_motherboard.serial,
                 ))

netbox_agent/lshw.py

@@ -86,19 +86,19 @@ class LSHW():
 
         elif "nvme" in obj["configuration"]["driver"]:
             nvme = json.loads(
-                            subprocess.check_output(["nvme", '-list', '-o', 'json'],
+                subprocess.check_output(
-                            encoding='utf8'))  # noqa: E128
+                    ["nvme", '-list', '-o', 'json'],
+                    encoding='utf8')
+            )
 
+            for device in nvme["Devices"]:
                 d = {}
-            d["vendor"] = obj["vendor"]
+                d['logicalname'] = device["DevicePath"]
-            d["version"] = obj["version"]
+                d['product'] = device["ModelNumber"]
-            d["product"] = obj["product"]
+                d['serial'] = device["SerialNumber"]
-
+                d["version"] = device["Firmware"]
+                d['size'] = device["UsedSize"]
                 d['description'] = "NVME Disk"
-            d['product'] = nvme["Devices"][0]["ModelNumber"]
-            d['size'] = nvme["Devices"][0]["PhysicalSize"]
-            d['serial'] = nvme["Devices"][0]["SerialNumber"]
-            d['logicalname'] = nvme["Devices"][0]["DevicePath"]
 
                 self.disks.append(d)
 
@@ -127,7 +127,7 @@ class LSHW():
             d["id"] = dimm.get("id")
             d["serial"] = dimm.get("serial", 'N/A')
             d["vendor"] = dimm.get("vendor", 'N/A')
-            d["product"] = dimm.get("product")
+            d["product"] = dimm.get("product", 'N/A')
             d["size"] = dimm.get("size", 0) / 2 ** 20 / 1024
 
             self.memories.append(d)

netbox_agent/network.py

@@ -24,20 +24,20 @@ class Network(object):
         self.nics = self.scan()
         self.ipmi = None
         self.dcim_choices = {}
-        dcim_c = nb.dcim.choices()
+        dcim_c = nb.dcim.interfaces.choices()
-
+        for _choice_type in dcim_c:
-        for choice in dcim_c:
+            key = 'interface:{}'.format(_choice_type)
-            self.dcim_choices[choice] = {}
+            self.dcim_choices[key] = {}
-            for c in dcim_c[choice]:
+            for choice in dcim_c[_choice_type]:
-                self.dcim_choices[choice][c['label']] = c['value']
+                self.dcim_choices[key][choice['display_name']] = choice['value']
 
         self.ipam_choices = {}
-        ipam_c = nb.ipam.choices()
+        ipam_c = nb.ipam.ip_addresses.choices()
-
+        for _choice_type in ipam_c:
-        for choice in ipam_c:
+            key = 'ip-address:{}'.format(_choice_type)
-            self.ipam_choices[choice] = {}
+            self.ipam_choices[key] = {}
-            for c in ipam_c[choice]:
+            for choice in ipam_c[_choice_type]:
-                self.ipam_choices[choice][c['label']] = c['value']
+                self.ipam_choices[key][choice['display_name']] = choice['value']
 
     def get_network_type():
         return NotImplementedError
@@ -93,6 +93,12 @@ class Network(object):
                 bonding_slaves = open(
                     '/sys/class/net/{}/bonding/slaves'.format(interface)
                 ).read().split()
+
+            # Tun and TAP support
+            virtual = os.path.isfile(
+                '/sys/class/net/{}/tun_flags'.format(interface)
+            )
+
             nic = {
                 'name': interface,
                 'mac': mac if mac != '00:00:00:00:00:00' else None,
@@ -103,6 +109,7 @@ class Network(object):
                     ) for x in ip_addr
                 ] if ip_addr else None,  # FIXME: handle IPv6 addresses
                 'ethtool': Ethtool(interface).parse(),
+                'virtual': virtual,
                 'vlan': vlan,
                 'bonding': bonding,
                 'bonding_slaves': bonding_slaves,
@@ -162,6 +169,10 @@ class Network(object):
 
         if nic.get('bonding'):
             return self.dcim_choices['interface:type']['Link Aggregation Group (LAG)']
+
+        if nic.get('virtual'):
+            return self.dcim_choices['interface:type']['Virtual']
+
         if nic.get('ethtool') is None:
             return self.dcim_choices['interface:type']['Other']
 
@@ -240,13 +251,18 @@ class Network(object):
             name=nic['name'], mac=nic['mac'], device=self.device.name))
 
         nb_vlan = None
-        interface = self.nb_net.interfaces.create(
+
-            name=nic['name'],
+        params = {
-            mac_address=nic['mac'],
+            'name': nic['name'],
-            type=type,
+            'type': type,
-            mgmt_only=mgmt,
+            'mgmt_only': mgmt,
             **self.custom_arg,
-        )
+        }
+
+        if not nic.get('virtual', False):
+            params['mac_address'] = nic['mac']
+
+        interface = self.nb_net.interfaces.create(**params)
 
         if nic['vlan']:
             nb_vlan = self.get_or_create_vlan(nic['vlan'])
@@ -442,8 +458,8 @@ class ServerNetwork(Network):
         self.server = server
         self.device = self.server.get_netbox_server()
         self.nb_net = nb.dcim
-        self.custom_arg = {'device': self.device.id}
+        self.custom_arg = {'device': getattr(self.device, "id", None)}
-        self.custom_arg_id = {'device_id': self.device.id}
+        self.custom_arg_id = {'device_id': getattr(self.device, "id", None)}
 
     def get_network_type(self):
         return 'server'
@@ -562,15 +578,15 @@ class VirtualNetwork(Network):
         self.server = server
         self.device = self.server.get_netbox_vm()
         self.nb_net = nb.virtualization
-        self.custom_arg = {'virtual_machine': self.device.id}
+        self.custom_arg = {'virtual_machine': getattr(self.device, "id", None)}
-        self.custom_arg_id = {'virtual_machine_id': self.device.id}
+        self.custom_arg_id = {'virtual_machine_id': getattr(self.device, "id", None)}
 
-        dcim_c = nb.virtualization.choices()
+        dcim_c = nb.virtualization.interfaces.choices()
-
+        for _choice_type in dcim_c:
-        for choice in dcim_c:
+            key = 'interface:{}'.format(_choice_type)
-            self.dcim_choices[choice] = {}
+            self.dcim_choices[key] = {}
-            for c in dcim_c[choice]:
+            for choice in dcim_c[_choice_type]:
-                self.dcim_choices[choice][c['label']] = c['value']
+                self.dcim_choices[key][choice['display_name']] = choice['value']
 
     def get_network_type(self):
         return 'virtual'

netbox_agent/power.py

@@ -29,8 +29,15 @@ class PowerSupply():
                 psu.get('Manufacturer', 'No Manufacturer').strip(),
                 psu.get('Name', 'No name').strip(),
             )
+
+            sn = psu.get('Serial Number', '').strip()
+            # Let's assume that if no serial and no power reported we skip it
+            if sn == '' and max_power is None:
+                continue
+            if sn == '':
+                sn = 'N/A'
             power_supply.append({
-                'name': psu.get('Serial Number', 'No S/N').strip(),
+                'name': sn,
                 'description': desc,
                 'allocated_draw': None,
                 'maximum_draw': max_power,

netbox_agent/raid/hp.py

@@ -54,6 +54,13 @@ def _get_dict(lines, start_index, indentation):
             continue
 
         current_line_indentation = _get_indentation(current_line)
+        # This check ignore some useless information that make
+        # crash the parsing
+        product_name = REGEXP_CONTROLLER_HP.search(current_line)
+        if current_line_indentation == 0 and not product_name:
+            i = i + 1
+            continue
+
         if current_line_indentation == indentation:
             current_item = current_line.lstrip(' ')
 

requirements.txt

@@ -1,5 +1,5 @@
 pynetbox==4.3.1
-netaddr==0.7.19
+netaddr==0.7.20
 netifaces==0.10.9
 pyyaml==5.3.1
-jsonargparse==2.25.3
+jsonargparse==2.31.0

setup.py

@@ -2,9 +2,10 @@ from setuptools import find_packages, setup
 
 setup(
     name='netbox_agent',
-    version='0.5.0',
+    version='0.6.1',
     description='NetBox agent for server',
     long_description=open('README.md', encoding="utf-8").read(),
+    long_description_content_type='text/markdown',
     url='https://github.com/solvik/netbox_agent',
     author='Solvik Blum',
     author_email='solvik@solvik.fr',
@@ -14,10 +15,10 @@ setup(
     use_scm_version=True,
     install_requires=[
         'pynetbox==4.3.1',
-        'netaddr==0.7.19',
+        'netaddr==0.7.20',
         'netifaces==0.10.9',
         'pyyaml==5.3.1',
-        'jsonargparse==2.25.3',
+        'jsonargparse==2.31.0',
     ],
     zip_safe=False,
     keywords=['netbox'],