major update to create separate reusable code

Created a shared transform.py and config_storcli.py files.
This commit is contained in:
Francisco Lopez 2018-08-13 11:33:30 -07:00 committed by GitHub
parent fa9a6234b7
commit 43ddbb6c64
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 289 additions and 17 deletions

View File

@ -1,2 +1,50 @@
# storcli-check
Use LSI storcli command line to check status of disk
## generate-master.py
This script is called manually and generates a "master" file to compare later
runs of storcli against. It calls transform.py to ignore some data that can
change between storcli runs.
## compare.py
This script is intended to be called by send-notice.sh. It compares a file
generated by generate-master.py against the output of a current run of the
storcli command. It calls transform.py to ignore some data that can change
between storcli runs.
## transform.py
This is a helper script called by generate-master.py and compare.py to change
some data that can change between runs but the changes themselves are not
considered important, i.e. a background task is active on one or more of the
RAID groups.
## send-notice.sh
This script is called directly from cron (can be called manually as well). It
is responsible for sending the output of the compare.py command to the e-mail
addresses specified in the script. If no problems are encountered, the script
will only send one message per day; it will send an alert message each time
invoked if a problem is detected.
Please change the default settings to match your environment.
## config_storcli.py
This configuration file defines the full path to the storcli command as well as
the necessary command-line arguments.
You can substitute other commands, i.e. "/bin/cat /dev/null" for testing
purposes.
### raid.master.txt
Sample output of storcli command for use when testing on a machine without an
appropriate RAID card or without root access.
### crontab.txt
Sample UNCX cron entry that runs hourly.

View File

@ -4,7 +4,10 @@ import difflib
import re
import subprocess
import sys
import tempfile
import time
import transform
import config_storcli
# If this is invoked from cron, let users know the name of the script.
@ -19,18 +22,16 @@ if len( sys.argv ) != 2:
mf = sys.argv[1]
fh = open( mf, "r" )
# Command to generate detailed disk information
run_cmd = [
"/opt/MegaRAID/storcli/storcli64",
"/c0/dALL",
"show",
"all" ]
diffCount = 0
pOutput = subprocess.Popen( run_cmd, stdout=subprocess.PIPE )
tmpfh = tempfile.TemporaryFile()
# Command defined in config_storcli.py file
pOutput = subprocess.Popen( config_storcli.cmd_detail, stdout=subprocess.PIPE )
transform.saveOutput( pOutput.stdout, tmpfh )
tmpfh.seek(0)
# compare the output from a previous run with output from the current run
for l in difflib.unified_diff( fh.readlines(), pOutput.stdout.readlines(), mf, tofiledate=time.ctime(), n=0 ):
for l in difflib.unified_diff( fh.readlines(), tmpfh.readlines(), mf, tofiledate=time.ctime(), n=0 ):
if diffCount == 0:
print "RAID Changes\n============"
print l.rstrip()
@ -41,15 +42,9 @@ diskString = "State :"
failString = "\APredictive"
saveString = ""
# Command to generate predictive failures
run_cmd = [
"/opt/MegaRAID/storcli/storcli64",
"/c0/eALL/sALL",
"show",
"all" ]
predictCount = 0
pOutput = subprocess.Popen( run_cmd, stdout=subprocess.PIPE )
# Command defined in config_storcli.py file
pOutput = subprocess.Popen( config_storcli.cmd_predict, stdout=subprocess.PIPE )
# look for predictive failures counts that are not zero
for line in pOutput.stdout:

23
config_storcli.py Normal file
View File

@ -0,0 +1,23 @@
# config-storcli.py
# Command strings to generate storcli reports
# Command to generate detailed disk information
cmd_detail = [
"/opt/MegaRAID/storcli/storcli64",
"/c0/dALL",
"show",
"all" ]
# for debugging
#cmd_detail = [ "/bin/cat", "raid.current.txt" ]
# Command to generate predictive failures
cmd_predict = [
"/opt/MegaRAID/storcli/storcli64",
"/c0/eALL/sALL",
"show",
"all" ]
# for debugging
#cmd_predict = [ "/bin/cat", "/dev/null" ]

2
crontab.txt Normal file
View File

@ -0,0 +1,2 @@
# Check raid status each hour, send status message once per day if no errors
25 * * * * /root/raid_check/send-notice.sh

18
generate-master.py Normal file
View File

@ -0,0 +1,18 @@
#! /usr/bin/python
import sys
import subprocess
import transform
import config_storcli
if len( sys.argv ) != 2:
print "Usage:", sys.argv[0], "storcli-output-file"
sys.exit(0)
# Command defined in config_storcli.py file
pOutput = subprocess.Popen( config_storcli.cmd_detail, stdout=subprocess.PIPE )
fw = open( sys.argv[1], "w")
transform.saveOutput( pOutput.stdout, fw)
fw.close()

158
raid.master.txt Normal file
View File

@ -0,0 +1,158 @@
CLI Version = 007.0606.0000.0000 Mar 20, 2018
Operating system = Linux 4.4.0-21-generic
Controller = 0
Status = Success
Description = Show Diskgroup Succeeded
TOPOLOGY :
========
-----------------------------------------------------------------------------
DG Arr Row EID:Slot DID Type State BT Size PDC PI SED DS3 FSpace TR
-----------------------------------------------------------------------------
0 - - - - Cac0 Optl N 372.093 GB dflt N N none N N
0 0 - - - RAID0 Optl N 372.093 GB dflt N N none N N
0 0 0 8:11 46 DRIVE Onln N 372.093 GB dflt N N none - N
1 - - - - RAID6 Optl N 109.152 TB dflt N N dflt N N
1 0 - - - RAID6 Optl N 109.152 TB dflt N N dflt N N
1 0 0 9:0 42 DRIVE Onln N 7.276 TB dflt N N dflt - N
1 0 1 9:1 14 DRIVE Onln N 7.276 TB dflt N N dflt - N
1 0 2 9:2 18 DRIVE Onln N 7.276 TB dflt N N dflt - N
1 0 3 9:3 38 DRIVE Onln N 7.276 TB dflt N N dflt - N
1 0 4 9:4 41 DRIVE Onln N 7.276 TB dflt N N dflt - N
1 0 5 9:5 23 DRIVE Onln N 7.276 TB dflt N N dflt - N
1 0 6 9:6 26 DRIVE Onln N 7.276 TB dflt N N dflt - N
1 0 7 9:7 21 DRIVE Onln N 7.276 TB dflt N N dflt - N
1 0 8 9:8 13 DRIVE Onln N 7.276 TB dflt N N dflt - N
1 0 9 9:9 15 DRIVE Onln N 7.276 TB dflt N N dflt - N
1 0 10 9:10 10 DRIVE Onln N 7.276 TB dflt N N dflt - N
1 0 11 9:11 43 DRIVE Onln N 7.276 TB dflt N N dflt - N
1 0 12 9:12 27 DRIVE Onln N 7.276 TB dflt N N dflt - N
1 0 13 9:13 24 DRIVE Onln N 7.276 TB dflt N N dflt - N
1 0 14 9:14 44 DRIVE Onln N 7.276 TB dflt N N dflt - N
1 0 15 9:15 28 DRIVE Onln N 7.276 TB dflt N N dflt - N
1 0 16 9:16 19 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 - - - - RAID6 Optl N 109.152 TB dflt N N dflt N N
2 0 - - - RAID6 Optl N 109.152 TB dflt N N dflt N N
2 0 0 9:17 40 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 0 1 9:18 22 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 0 2 9:19 29 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 0 3 9:20 25 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 0 4 9:21 37 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 0 5 9:22 34 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 0 6 9:23 35 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 0 7 8:0 12 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 0 8 8:1 17 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 0 9 8:2 32 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 0 10 8:3 20 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 0 11 8:4 16 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 0 12 8:5 36 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 0 13 8:6 11 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 0 14 8:7 39 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 0 15 8:8 31 DRIVE Onln N 7.276 TB dflt N N dflt - N
2 0 16 8:9 33 DRIVE Onln N 7.276 TB dflt N N dflt - N
- - - 8:10 30 DRIVE GHS - 7.276 TB - - - - - N
-----------------------------------------------------------------------------
DG=Disk Group Index|Arr=Array Index|Row=Row Index|EID=Enclosure Device ID
DID=Device ID|Type=Drive Type|Onln=Online|Rbld=Rebuild|Dgrd=Degraded
Pdgd=Partially degraded|Offln=Offline|BT=Background Task Active
PDC=PD Cache|PI=Protection Info|SED=Self Encrypting Drive|Frgn=Foreign
DS3=Dimmer Switch 3|dflt=Default|Msng=Missing|FSpace=Free Space Present
TR=Transport Ready
VD LIST :
=======
---------------------------------------------------------------
DG/VD TYPE State Access Consist Cache Cac sCC Size Name
---------------------------------------------------------------
0/0 Cac0 Optl RW Yes RWTD - ON 372.093 GB
1/1 RAID6 Optl RW Yes RWBD - ON 109.152 TB
2/2 RAID6 Optl RW Yes RWBD - ON 109.152 TB
---------------------------------------------------------------
Cac=CacheCade|Rec=Recovery|OfLn=OffLine|Pdgd=Partially Degraded|Dgrd=Degraded
Optl=Optimal|RO=Read Only|RW=Read Write|HD=Hidden|TRANS=TransportReady|B=Blocked|
Consist=Consistent|R=Read Ahead Always|NR=No Read Ahead|WB=WriteBack|
AWB=Always WriteBack|WT=WriteThrough|C=Cached IO|D=Direct IO|sCC=Scheduled
Check Consistency
Total VD Count = 3
DG Drive LIST :
=============
---------------------------------------------------------------------------------
EID:Slt DID State DG Size Intf Med SED PI SeSz Model Sp Type
---------------------------------------------------------------------------------
8:11 46 Onln 0 372.093 GB SATA SSD N N 512B INTEL SSDSC2BA400G4 U -
9:0 42 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:1 14 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:2 18 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:3 38 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:4 41 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:5 23 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:6 26 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:7 21 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:8 13 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:9 15 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:10 10 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:11 43 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:12 27 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:13 24 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:14 44 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:15 28 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:16 19 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:17 40 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:18 22 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:19 29 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:20 25 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:21 37 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:22 34 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
9:23 35 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
8:0 12 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
8:1 17 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
8:2 32 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
8:3 20 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
8:4 16 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
8:5 36 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
8:6 11 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
8:7 39 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
8:8 31 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
8:9 33 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
---------------------------------------------------------------------------------
EID-Enclosure Device ID|Slt-Slot No.|DID-Device ID|DG-DriveGroup
DHS-Dedicated Hot Spare|UGood-Unconfigured Good|GHS-Global Hotspare
UBad-Unconfigured Bad|Onln-Online|Offln-Offline|Intf-Interface
Med-Media Type|SED-Self Encryptive Drive|PI-Protection Info
SeSz-Sector Size|Sp-Spun|U-Up|D-Down/PowerSave|T-Transition|F-Foreign
UGUnsp-Unsupported|UGShld-UnConfigured shielded|HSPShld-Hotspare shielded
CFShld-Configured shielded|Cpybck-CopyBack|CBShld-Copyback Shielded
Total Drive Count = 35
UN-CONFIGURED DRIVE LIST :
========================
----------------------------------------------------------------------------
EID:Slt DID State DG Size Intf Med SED PI SeSz Model Sp Type
----------------------------------------------------------------------------
8:10 30 GHS - 7.276 TB SAS HDD N N 512B HUH728080AL5200 D -
----------------------------------------------------------------------------
EID-Enclosure Device ID|Slt-Slot No.|DID-Device ID|DG-DriveGroup
DHS-Dedicated Hot Spare|UGood-Unconfigured Good|GHS-Global Hotspare
UBad-Unconfigured Bad|Onln-Online|Offln-Offline|Intf-Interface
Med-Media Type|SED-Self Encryptive Drive|PI-Protection Info
SeSz-Sector Size|Sp-Spun|U-Up|D-Down/PowerSave|T-Transition|F-Foreign
UGUnsp-Unsupported|UGShld-UnConfigured shielded|HSPShld-Hotspare shielded
CFShld-Configured shielded|Cpybck-CopyBack|CBShld-Copyback Shielded
Unconfigured Drive Count = 1

View File

@ -3,7 +3,6 @@
mailto="lopez@uci.edu"
errorsto="lopez@uci.edu"
scriptdir="/root/raid_check"
# previous good run of storcli to compare against current output
masterfile="$scriptdir/raid.master.txt"
subject="`hostname` RAID check `date +%Y-%m-%d`"
sendemail="n"

29
transform.py Normal file
View File

@ -0,0 +1,29 @@
import re
def saveOutput( readfh, writefh ):
convertDriveStatus = False
separatorCount = 0
isTopologyHeader = re.compile( "DG Arr Row EID:Slot DID Type State BT Size PDC PI SED DS3 FSpace TR " )
isUnconfiguredHeader = re.compile( "EID:Slt DID State DG Size Intf Med SED PI SeSz Model Sp Type " )
isSeparator = re.compile( "^-+$" )
for line in readfh.readlines():
if convertDriveStatus:
if isSeparator.match( line ):
if separatorCount == 1:
separatorCount = 0
convertDriveStatus = False
else:
separatorCount += 1
else:
if line[36] == "Y":
# set background task status to N for topology report
line = line[0:36] + "N" + line[37:]
elif line[68] == "U":
# set drive spun status to D for un-configured drives report
line = line[0:68] + "D" + line[69:]
else:
if isTopologyHeader.match( line ) or isUnconfiguredHeader.match( line ):
convertDriveStatus = True
writefh.write( line )