Added disks extended attributes
This patch brings some of the physical and virtual drive attributes as
`custom_fields` to the disks inventory.
The goal is to have this information present to ease disks maintenance
when a drive becomes unavailable and its attributes can't be read anymore
from the RAID controller.
It also helps to standardize the extended disk attributes across the
different manufacturers.
As the disk physical identifers were not available under the correct
format (hexadecimal format using the `xml` output as opposed as `X:Y:Z` format
using the default `list` format), the command line parser has been
refactored to read the `list` format, rather than `xml` one in the
`omreport` raid controller parser.
As the custom fields have to be created prior being able to register
the disks extended attributes, this feature is only activated using the
`--process-virtual-drives` command line parameter, or by setting
`process_virtual_drives` to `true` in the configuration file.
The custom fields to create as `DCIM > inventory item` `Text` are described
below.
NAME LABEL DESCRIPTION
mount_point Mount point Device mount point(s)
pd_identifier Physical disk identifier Physical disk identifier in the RAID controller
vd_array Virtual drive array Virtual drive array the disk is member of
vd_consistency Virtual drive consistency Virtual disk array consistency
vd_device Virtual drive device Virtual drive system device
vd_raid_type Virtual drive RAID Virtual drive array RAID type
vd_size Virtual drive size Virtual drive array size
In the current implementation, the disks attributes ore not updated: if
a disk with the correct serial number is found, it's sufficient to
consider it as up to date.
To force the reprocessing of the disks extended attributes, the
`--force-disk-refresh` command line option can be used: it removes all
existing disks to before populating them with the correct parsing.
Unless this option is specified, the extended attributes won't be
modified unless a disk is replaced.
It is possible to dump the physical/virtual disks map on the filesystem under
the JSON notation to ease or automate disks management. The file path has to
be provided using the `--dump-disks-map` command line parameter.
This commit is contained in:
Christophe Simon
parent
af9df9ab4b
commit
e789619b34
12 changed files with 504 additions and 222 deletions
|
|
@ -1,13 +1,20 @@
|
|||
import re
|
||||
import subprocess
|
||||
|
||||
from netbox_agent.config import config
|
||||
from netbox_agent.misc import get_vendor
|
||||
from netbox_agent.raid.base import Raid, RaidController
|
||||
from netbox_agent.misc import get_vendor
|
||||
from netbox_agent.config import config
|
||||
import subprocess
|
||||
import logging
|
||||
import re
|
||||
|
||||
REGEXP_CONTROLLER_HP = re.compile(r'Smart Array ([a-zA-Z0-9- ]+) in Slot ([0-9]+)')
|
||||
|
||||
|
||||
def ssacli(command):
|
||||
output = subprocess.getoutput('ssacli {}'.format(command) )
|
||||
lines = output.split('\n')
|
||||
lines = list(filter(None, lines))
|
||||
return lines
|
||||
|
||||
|
||||
def _parse_ctrl_output(lines):
|
||||
controllers = {}
|
||||
current_ctrl = None
|
||||
|
|
@ -18,11 +25,11 @@ def _parse_ctrl_output(lines):
|
|||
ctrl = REGEXP_CONTROLLER_HP.search(line)
|
||||
if ctrl is not None:
|
||||
current_ctrl = ctrl.group(1)
|
||||
controllers[current_ctrl] = {"Slot": ctrl.group(2)}
|
||||
if "Embedded" not in line:
|
||||
controllers[current_ctrl]["External"] = True
|
||||
controllers[current_ctrl] = {'Slot': ctrl.group(2)}
|
||||
if 'Embedded' not in line:
|
||||
controllers[current_ctrl]['External'] = True
|
||||
continue
|
||||
attr, val = line.split(": ", 1)
|
||||
attr, val = line.split(': ', 1)
|
||||
attr = attr.strip()
|
||||
val = val.strip()
|
||||
controllers[current_ctrl][attr] = val
|
||||
|
|
@ -39,27 +46,54 @@ def _parse_pd_output(lines):
|
|||
if not line or line.startswith('Note:'):
|
||||
continue
|
||||
# Parses the Array the drives are in
|
||||
if line.startswith("Array"):
|
||||
if line.startswith('Array'):
|
||||
current_array = line.split(None, 1)[1]
|
||||
# Detects new physical drive
|
||||
if line.startswith("physicaldrive"):
|
||||
if line.startswith('physicaldrive'):
|
||||
current_drv = line.split(None, 1)[1]
|
||||
drives[current_drv] = {}
|
||||
if current_array is not None:
|
||||
drives[current_drv]["Array"] = current_array
|
||||
drives[current_drv]['Array'] = current_array
|
||||
continue
|
||||
if ": " not in line:
|
||||
if ': ' not in line:
|
||||
continue
|
||||
attr, val = line.split(": ", 1)
|
||||
attr, val = line.split(': ', 1)
|
||||
drives.setdefault(current_drv, {})[attr] = val
|
||||
return drives
|
||||
|
||||
|
||||
def _parse_ld_output(lines):
|
||||
drives = {}
|
||||
current_array = None
|
||||
current_drv = None
|
||||
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line or line.startswith('Note:'):
|
||||
continue
|
||||
# Parses the Array the drives are in
|
||||
if line.startswith('Array'):
|
||||
current_array = line.split(None, 1)[1]
|
||||
drives[current_array] = {}
|
||||
# Detects new physical drive
|
||||
if line.startswith('Logical Drive'):
|
||||
current_drv = line.split(': ', 1)[1]
|
||||
drives.setdefault(current_array, {})['LogicalDrive'] = current_drv
|
||||
continue
|
||||
if ': ' not in line:
|
||||
continue
|
||||
attr, val = line.split(': ', 1)
|
||||
drives.setdefault(current_array, {})[attr] = val
|
||||
return drives
|
||||
|
||||
|
||||
class HPRaidController(RaidController):
|
||||
def __init__(self, controller_name, data):
|
||||
self.controller_name = controller_name
|
||||
self.data = data
|
||||
self.drives = self._get_physical_disks()
|
||||
self.pdrives = self._get_physical_disks()
|
||||
self.ldrives = self._get_logical_drives()
|
||||
self._get_virtual_drives_map()
|
||||
|
||||
def get_product_name(self):
|
||||
return self.controller_name
|
||||
|
|
@ -77,15 +111,12 @@ class HPRaidController(RaidController):
|
|||
return self.data.get('External', False)
|
||||
|
||||
def _get_physical_disks(self):
|
||||
output = subprocess.getoutput(
|
||||
'ssacli ctrl slot={slot} pd all show detail'.format(slot=self.data['Slot'])
|
||||
)
|
||||
lines = output.split('\n')
|
||||
lines = list(filter(None, lines))
|
||||
drives = _parse_pd_output(lines)
|
||||
ret = []
|
||||
lines = ssacli('ctrl slot={} pd all show detail'.format(self.data['Slot']))
|
||||
pdrives = _parse_pd_output(lines)
|
||||
ret = {}
|
||||
|
||||
for name, attrs in drives.items():
|
||||
for name, attrs in pdrives.items():
|
||||
array = attrs.get('Array', '')
|
||||
model = attrs.get('Model', '').strip()
|
||||
vendor = None
|
||||
if model.startswith('HP'):
|
||||
|
|
@ -95,7 +126,8 @@ class HPRaidController(RaidController):
|
|||
else:
|
||||
vendor = get_vendor(model)
|
||||
|
||||
ret.append({
|
||||
ret[name] = {
|
||||
'Array': array,
|
||||
'Model': model,
|
||||
'Vendor': vendor,
|
||||
'SN': attrs.get('Serial Number', '').strip(),
|
||||
|
|
@ -103,11 +135,40 @@ class HPRaidController(RaidController):
|
|||
'Type': 'SSD' if attrs.get('Interface Type') == 'Solid State SATA'
|
||||
else 'HDD',
|
||||
'_src': self.__class__.__name__,
|
||||
})
|
||||
}
|
||||
return ret
|
||||
|
||||
def _get_logical_drives(self):
|
||||
lines = ssacli('ctrl slot={} ld all show detail'.format(self.data['Slot']))
|
||||
ldrives = _parse_ld_output(lines)
|
||||
ret = {}
|
||||
|
||||
for array, attrs in ldrives.items():
|
||||
ret[array] = {
|
||||
'vd_array': array,
|
||||
'vd_size': attrs['Size'],
|
||||
'vd_consistency': attrs['Status'],
|
||||
'vd_raid_type': 'RAID {}'.format(attrs['Fault Tolerance']),
|
||||
'vd_device': attrs['LogicalDrive'],
|
||||
'mount_point': attrs['Mount Points']
|
||||
}
|
||||
return ret
|
||||
|
||||
def _get_virtual_drives_map(self):
|
||||
for name, attrs in self.pdrives.items():
|
||||
array = attrs["Array"]
|
||||
ld = self.ldrives.get(array)
|
||||
if ld is None:
|
||||
logging.error(
|
||||
"Failed to find array information for physical drive {}."
|
||||
" Ignoring.".format(name)
|
||||
)
|
||||
continue
|
||||
attrs['custom_fields'] = ld
|
||||
attrs['custom_fields']['pd_identifier'] = name
|
||||
|
||||
def get_physical_disks(self):
|
||||
return self.drives
|
||||
return list(self.pdrives.values())
|
||||
|
||||
|
||||
class HPRaid(Raid):
|
||||
|
|
|
|||
|
|
@ -1,25 +1,32 @@
|
|||
import re
|
||||
import subprocess
|
||||
import xml.etree.ElementTree as ET # NOQA
|
||||
|
||||
from netbox_agent.misc import get_vendor
|
||||
from netbox_agent.raid.base import Raid, RaidController
|
||||
|
||||
# Inspiration from https://github.com/asciiphil/perc-status/blob/master/perc-status
|
||||
from netbox_agent.misc import get_vendor, get_mount_points
|
||||
from netbox_agent.config import config
|
||||
import subprocess
|
||||
import logging
|
||||
import re
|
||||
|
||||
|
||||
def get_field(obj, fieldname):
|
||||
f = obj.find(fieldname)
|
||||
if f is None:
|
||||
return None
|
||||
if f.attrib['type'] in ['u32', 'u64']:
|
||||
if re.search('Mask$', fieldname):
|
||||
return int(f.text, 2)
|
||||
else:
|
||||
return int(f.text)
|
||||
if f.attrib['type'] == 'astring':
|
||||
return f.text
|
||||
return f.text
|
||||
def omreport(sub_command):
|
||||
command = 'omreport {}'.format(sub_command)
|
||||
output = subprocess.getoutput(command)
|
||||
res = {}
|
||||
section_re = re.compile('^[A-Z]')
|
||||
current_section = None
|
||||
current_obj = None
|
||||
|
||||
for line in output.split('\n'):
|
||||
if ': ' in line:
|
||||
attr, value = line.split(': ', 1)
|
||||
attr = attr.strip()
|
||||
value = value.strip()
|
||||
if attr == 'ID':
|
||||
obj = {}
|
||||
res.setdefault(current_section, []).append(obj)
|
||||
current_obj = obj
|
||||
current_obj[attr] = value
|
||||
elif section_re.search(line) is not None:
|
||||
current_section = line.strip()
|
||||
return res
|
||||
|
||||
|
||||
class OmreportController(RaidController):
|
||||
|
|
@ -28,49 +35,88 @@ class OmreportController(RaidController):
|
|||
self.controller_index = controller_index
|
||||
|
||||
def get_product_name(self):
|
||||
return get_field(self.data, 'Name')
|
||||
return self.data['Name']
|
||||
|
||||
def get_manufacturer(self):
|
||||
return None
|
||||
|
||||
def get_serial_number(self):
|
||||
return get_field(self.data, 'DeviceSerialNumber')
|
||||
return self.data.get('DeviceSerialNumber')
|
||||
|
||||
def get_firmware_version(self):
|
||||
return get_field(self.data, 'Firmware Version')
|
||||
return self.data.get('Firmware Version')
|
||||
|
||||
def _get_physical_disks(self):
|
||||
pds = {}
|
||||
res = omreport('storage pdisk controller={}'.format(
|
||||
self.controller_index
|
||||
))
|
||||
for pdisk in [d for d in list(res.values())[0]]:
|
||||
disk_id = pdisk['ID']
|
||||
size = re.sub('B .*$', 'B', pdisk['Capacity'])
|
||||
pds[disk_id] = {
|
||||
'Vendor': get_vendor(pdisk['Vendor ID']),
|
||||
'Model': pdisk['Product ID'],
|
||||
'SN': pdisk['Serial No.'],
|
||||
'Size': size,
|
||||
'Type': pdisk['Media'],
|
||||
'_src': self.__class__.__name__,
|
||||
}
|
||||
return pds
|
||||
|
||||
def _get_virtual_drives_map(self):
|
||||
pds = {}
|
||||
res = omreport('storage vdisk controller={}'.format(
|
||||
self.controller_index
|
||||
))
|
||||
for vdisk in [d for d in list(res.values())[0]]:
|
||||
vdisk_id = vdisk['ID']
|
||||
device = vdisk['Device Name']
|
||||
mount_points = get_mount_points()
|
||||
mp = mount_points.get(device, 'n/a')
|
||||
size = re.sub('B .*$', 'B', vdisk['Size'])
|
||||
vd = {
|
||||
'vd_array': vdisk_id,
|
||||
'vd_size': size,
|
||||
'vd_consistency': vdisk['State'],
|
||||
'vd_raid_type': vdisk['Layout'],
|
||||
'vd_device': vdisk['Device Name'],
|
||||
'mount_point': ', '.join(sorted(mp)),
|
||||
}
|
||||
drives_res = omreport(
|
||||
'storage pdisk controller={} vdisk={}'.format(
|
||||
self.controller_index, vdisk_id
|
||||
))
|
||||
for pdisk in [d for d in list(drives_res.values())[0]]:
|
||||
pds[pdisk['ID']] = vd
|
||||
return pds
|
||||
|
||||
def get_physical_disks(self):
|
||||
ret = []
|
||||
output = subprocess.getoutput(
|
||||
'omreport storage controller controller={} -fmt xml'.format(self.controller_index)
|
||||
)
|
||||
root = ET.fromstring(output)
|
||||
et_array_disks = root.find('ArrayDisks')
|
||||
if et_array_disks is not None:
|
||||
for obj in et_array_disks.findall('DCStorageObject'):
|
||||
ret.append({
|
||||
'Vendor': get_vendor(get_field(obj, 'Vendor')),
|
||||
'Model': get_field(obj, 'ProductID'),
|
||||
'SN': get_field(obj, 'DeviceSerialNumber'),
|
||||
'Size': '{:.0f}GB'.format(
|
||||
int(get_field(obj, 'Length')) / 1024 / 1024 / 1024
|
||||
),
|
||||
'Type': 'HDD' if int(get_field(obj, 'MediaType')) == 1 else 'SSD',
|
||||
'_src': self.__class__.__name__,
|
||||
})
|
||||
return ret
|
||||
pds = self._get_physical_disks()
|
||||
vds = self._get_virtual_drives_map()
|
||||
for pd_identifier, vd in vds.items():
|
||||
if pd_identifier not in pds:
|
||||
logging.error(
|
||||
'Physical drive {} listed in virtual drive {} not '
|
||||
'found in drives list'.format(
|
||||
pd_identifier, vd['vd_array']
|
||||
)
|
||||
)
|
||||
continue
|
||||
pds[pd_identifier].setdefault('custom_fields', {}).update(vd)
|
||||
pds[pd_identifier]['custom_fields']['pd_identifier'] = pd_identifier
|
||||
return list(pds.values())
|
||||
|
||||
|
||||
class OmreportRaid(Raid):
|
||||
def __init__(self):
|
||||
output = subprocess.getoutput('omreport storage controller -fmt xml')
|
||||
controller_xml = ET.fromstring(output)
|
||||
self.controllers = []
|
||||
res = omreport('storage controller')
|
||||
|
||||
for obj in controller_xml.find('Controllers').findall('DCStorageObject'):
|
||||
ctrl_index = get_field(obj, 'ControllerNum')
|
||||
for controller in res['Controller']:
|
||||
ctrl_index = controller['ID']
|
||||
self.controllers.append(
|
||||
OmreportController(ctrl_index, obj)
|
||||
OmreportController(ctrl_index, controller)
|
||||
)
|
||||
|
||||
def get_controllers(self):
|
||||
|
|
|
|||
|
|
@ -1,8 +1,31 @@
|
|||
import json
|
||||
import subprocess
|
||||
|
||||
from netbox_agent.misc import get_vendor
|
||||
from netbox_agent.raid.base import Raid, RaidController
|
||||
from netbox_agent.misc import get_vendor, get_mount_points
|
||||
from netbox_agent.config import config
|
||||
import subprocess
|
||||
import logging
|
||||
import json
|
||||
import re
|
||||
import os
|
||||
|
||||
|
||||
def storecli(sub_command):
|
||||
command = 'storcli {} J'.format(sub_command)
|
||||
output = subprocess.getoutput(command)
|
||||
data = json.loads(output)
|
||||
controllers = dict([
|
||||
(
|
||||
c['Command Status']['Controller'],
|
||||
c['Response Data']
|
||||
) for c in data['Controllers']
|
||||
if c['Command Status']['Status'] == 'Success'
|
||||
])
|
||||
if not controllers:
|
||||
logging.error(
|
||||
"Failed to execute command '{}'. "
|
||||
"Ignoring data.".format(command)
|
||||
)
|
||||
return {}
|
||||
return controllers
|
||||
|
||||
|
||||
class StorcliController(RaidController):
|
||||
|
|
@ -22,52 +45,101 @@ class StorcliController(RaidController):
|
|||
def get_firmware_version(self):
|
||||
return self.data['FW Package Build']
|
||||
|
||||
def get_physical_disks(self):
|
||||
ret = []
|
||||
output = subprocess.getoutput(
|
||||
'storcli /c{}/eall/sall show all J'.format(self.controller_index)
|
||||
)
|
||||
drive_infos = json.loads(output)['Controllers'][self.controller_index]['Response Data']
|
||||
def _get_physical_disks(self):
|
||||
pds = {}
|
||||
cmd = '/c{}/eall/sall show all'.format(self.controller_index)
|
||||
controllers = storecli(cmd)
|
||||
pd_info = controllers[self.controller_index]
|
||||
pd_re = re.compile(r'^Drive (/c\d+/e\d+/s\d+)$')
|
||||
|
||||
for physical_drive in self.data['PD LIST']:
|
||||
enclosure = physical_drive.get('EID:Slt').split(':')[0]
|
||||
slot = physical_drive.get('EID:Slt').split(':')[1]
|
||||
size = physical_drive.get('Size').strip()
|
||||
media_type = physical_drive.get('Med').strip()
|
||||
drive_identifier = 'Drive /c{}/e{}/s{}'.format(
|
||||
str(self.controller_index), str(enclosure), str(slot)
|
||||
)
|
||||
drive_attr = drive_infos['{} - Detailed Information'.format(drive_identifier)][
|
||||
'{} Device attributes'.format(drive_identifier)]
|
||||
model = drive_attr.get('Model Number', '').strip()
|
||||
ret.append({
|
||||
for section, attrs in pd_info.items():
|
||||
reg = pd_re.search(section)
|
||||
if reg is None:
|
||||
continue
|
||||
pd_name = reg.group(1)
|
||||
pd_attr = attrs[0]
|
||||
pd_identifier = pd_attr['EID:Slt']
|
||||
size = pd_attr.get('Size', '').strip()
|
||||
media_type = pd_attr.get('Med', '').strip()
|
||||
pd_details = pd_info['{} - Detailed Information'.format(section)]
|
||||
pd_dev_attr = pd_details['{} Device attributes'.format(section)]
|
||||
model = pd_dev_attr.get('Model Number', '').strip()
|
||||
pd = {
|
||||
'Model': model,
|
||||
'Vendor': get_vendor(model),
|
||||
'SN': drive_attr.get('SN', '').strip(),
|
||||
'SN': pd_dev_attr.get('SN', '').strip(),
|
||||
'Size': size,
|
||||
'Type': media_type,
|
||||
'_src': self.__class__.__name__,
|
||||
})
|
||||
return ret
|
||||
}
|
||||
if config.process_virtual_drives:
|
||||
pd.setdefault('custom_fields', {})['pd_identifier'] = pd_name
|
||||
pds[pd_identifier] = pd
|
||||
return pds
|
||||
|
||||
def _get_virtual_drives_map(self):
|
||||
vds = {}
|
||||
cmd = '/c{}/vall show all'.format(self.controller_index)
|
||||
controllers = storecli(cmd)
|
||||
vd_info = controllers[self.controller_index]
|
||||
mount_points = get_mount_points()
|
||||
|
||||
for vd_identifier, vd_attrs in vd_info.items():
|
||||
if not vd_identifier.startswith("/c{}/v".format(self.controller_index)):
|
||||
continue
|
||||
volume = vd_identifier.split("/")[-1].lstrip("v")
|
||||
vd_attr = vd_attrs[0]
|
||||
vd_pd_identifier = 'PDs for VD {}'.format(volume)
|
||||
vd_pds = vd_info[vd_pd_identifier]
|
||||
vd_prop_identifier = 'VD{} Properties'.format(volume)
|
||||
vd_properties = vd_info[vd_prop_identifier]
|
||||
for pd in vd_pds:
|
||||
pd_identifier = pd["EID:Slt"]
|
||||
wwn = vd_properties["SCSI NAA Id"]
|
||||
wwn_path = "/dev/disk/by-id/wwn-0x{}".format(wwn)
|
||||
device = os.path.realpath(wwn_path)
|
||||
mp = mount_points.get(device, "n/a")
|
||||
vds[pd_identifier] = {
|
||||
"vd_array": vd_identifier,
|
||||
"vd_size": vd_attr["Size"],
|
||||
"vd_consistency": vd_attr["Consist"],
|
||||
"vd_raid_type": vd_attr["TYPE"],
|
||||
"vd_device": device,
|
||||
"mount_point": ", ".join(sorted(mp))
|
||||
}
|
||||
return vds
|
||||
|
||||
def get_physical_disks(self):
|
||||
# Parses physical disks information
|
||||
pds = self._get_physical_disks()
|
||||
|
||||
# Parses virtual drives information and maps them to physical disks
|
||||
vds = self._get_virtual_drives_map()
|
||||
for pd_identifier, vd in vds.items():
|
||||
if pd_identifier not in pds:
|
||||
logging.error(
|
||||
"Physical drive {} listed in virtual drive {} not "
|
||||
"found in drives list".format(
|
||||
pd_identifier, vd["vd_array"]
|
||||
)
|
||||
)
|
||||
continue
|
||||
pds[pd_identifier].setdefault("custom_fields", {}).update(vd)
|
||||
|
||||
return list(pds.values())
|
||||
|
||||
|
||||
class StorcliRaid(Raid):
|
||||
def __init__(self):
|
||||
self.output = subprocess.getoutput('storcli /call show J')
|
||||
self.data = json.loads(self.output)
|
||||
self.controllers = []
|
||||
|
||||
if len([
|
||||
x for x in self.data['Controllers']
|
||||
if x['Command Status']['Status'] == 'Success'
|
||||
]) > 0:
|
||||
for controller in self.data['Controllers']:
|
||||
self.controllers.append(
|
||||
StorcliController(
|
||||
controller['Command Status']['Controller'],
|
||||
controller['Response Data']
|
||||
)
|
||||
controllers = storecli('/call show')
|
||||
for controller_id, controller_data in controllers.items():
|
||||
self.controllers.append(
|
||||
StorcliController(
|
||||
controller_id,
|
||||
controller_data
|
||||
)
|
||||
)
|
||||
|
||||
def get_controllers(self):
|
||||
return self.controllers
|
||||
|
|
|
|||
Loading…
Add table
editor.link_modal.header
Reference in a new issue