major update to create separate reusable code
Created a shared transform.py and config_storcli.py files.
This commit is contained in:
parent
fa9a6234b7
commit
43ddbb6c64
48
README.md
48
README.md
|
@ -1,2 +1,50 @@
|
|||
# storcli-check
|
||||
|
||||
Use LSI storcli command line to check status of disk
|
||||
|
||||
## generate-master.py
|
||||
|
||||
This script is called manually and generates a "master" file to compare later
|
||||
runs of storcli against. It calls transform.py to ignore some data that can
|
||||
change between storcli runs.
|
||||
|
||||
## compare.py
|
||||
|
||||
This script is intended to be called by send-notice.sh. It compares a file
|
||||
generated by generate-master.py against the output of a current run of the
|
||||
storcli command. It calls transform.py to ignore some data that can change
|
||||
between storcli runs.
|
||||
|
||||
## transform.py
|
||||
|
||||
This is a helper script called by generate-master.py and compare.py to change
|
||||
some data that can change between runs but the changes themselves are not
|
||||
considered important, i.e. a background task is active on one or more of the
|
||||
RAID groups.
|
||||
|
||||
## send-notice.sh
|
||||
|
||||
This script is called directly from cron (can be called manually as well). It
|
||||
is responsible for sending the output of the compare.py command to the e-mail
|
||||
addresses specified in the script. If no problems are encountered, the script
|
||||
will only send one message per day; it will send an alert message each time
|
||||
invoked if a problem is detected.
|
||||
|
||||
Please change the default settings to match your environment.
|
||||
|
||||
## config_storcli.py
|
||||
|
||||
This configuration file defines the full path to the storcli command as well as
|
||||
the necessary command-line arguments.
|
||||
|
||||
You can substitute other commands, i.e. "/bin/cat /dev/null" for testing
|
||||
purposes.
|
||||
|
||||
### raid.master.txt
|
||||
|
||||
Sample output of storcli command for use when testing on a machine without an
|
||||
appropriate RAID card or without root access.
|
||||
|
||||
### crontab.txt
|
||||
|
||||
Sample UNCX cron entry that runs hourly.
|
||||
|
|
27
compare.py
27
compare.py
|
@ -4,7 +4,10 @@ import difflib
|
|||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import transform
|
||||
import config_storcli
|
||||
|
||||
|
||||
# If this is invoked from cron, let users know the name of the script.
|
||||
|
@ -19,18 +22,16 @@ if len( sys.argv ) != 2:
|
|||
mf = sys.argv[1]
|
||||
fh = open( mf, "r" )
|
||||
|
||||
# Command to generate detailed disk information
|
||||
run_cmd = [
|
||||
"/opt/MegaRAID/storcli/storcli64",
|
||||
"/c0/dALL",
|
||||
"show",
|
||||
"all" ]
|
||||
|
||||
diffCount = 0
|
||||
pOutput = subprocess.Popen( run_cmd, stdout=subprocess.PIPE )
|
||||
tmpfh = tempfile.TemporaryFile()
|
||||
# Command defined in config_storcli.py file
|
||||
pOutput = subprocess.Popen( config_storcli.cmd_detail, stdout=subprocess.PIPE )
|
||||
transform.saveOutput( pOutput.stdout, tmpfh )
|
||||
tmpfh.seek(0)
|
||||
|
||||
# compare the output from a previous run with output from the current run
|
||||
for l in difflib.unified_diff( fh.readlines(), pOutput.stdout.readlines(), mf, tofiledate=time.ctime(), n=0 ):
|
||||
for l in difflib.unified_diff( fh.readlines(), tmpfh.readlines(), mf, tofiledate=time.ctime(), n=0 ):
|
||||
if diffCount == 0:
|
||||
print "RAID Changes\n============"
|
||||
print l.rstrip()
|
||||
|
@ -41,15 +42,9 @@ diskString = "State :"
|
|||
failString = "\APredictive"
|
||||
saveString = ""
|
||||
|
||||
# Command to generate predictive failures
|
||||
run_cmd = [
|
||||
"/opt/MegaRAID/storcli/storcli64",
|
||||
"/c0/eALL/sALL",
|
||||
"show",
|
||||
"all" ]
|
||||
|
||||
predictCount = 0
|
||||
pOutput = subprocess.Popen( run_cmd, stdout=subprocess.PIPE )
|
||||
# Command defined in config_storcli.py file
|
||||
pOutput = subprocess.Popen( config_storcli.cmd_predict, stdout=subprocess.PIPE )
|
||||
|
||||
# look for predictive failures counts that are not zero
|
||||
for line in pOutput.stdout:
|
||||
|
|
|
@ -0,0 +1,23 @@
|
|||
# config-storcli.py
|
||||
# Command strings to generate storcli reports
|
||||
|
||||
|
||||
# Command to generate detailed disk information
|
||||
cmd_detail = [
|
||||
"/opt/MegaRAID/storcli/storcli64",
|
||||
"/c0/dALL",
|
||||
"show",
|
||||
"all" ]
|
||||
# for debugging
|
||||
#cmd_detail = [ "/bin/cat", "raid.current.txt" ]
|
||||
|
||||
|
||||
# Command to generate predictive failures
|
||||
cmd_predict = [
|
||||
"/opt/MegaRAID/storcli/storcli64",
|
||||
"/c0/eALL/sALL",
|
||||
"show",
|
||||
"all" ]
|
||||
# for debugging
|
||||
#cmd_predict = [ "/bin/cat", "/dev/null" ]
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
# Check raid status each hour, send status message once per day if no errors
|
||||
25 * * * * /root/raid_check/send-notice.sh
|
|
@ -0,0 +1,18 @@
|
|||
#! /usr/bin/python
|
||||
|
||||
import sys
|
||||
import subprocess
|
||||
import transform
|
||||
import config_storcli
|
||||
|
||||
|
||||
if len( sys.argv ) != 2:
|
||||
print "Usage:", sys.argv[0], "storcli-output-file"
|
||||
sys.exit(0)
|
||||
|
||||
# Command defined in config_storcli.py file
|
||||
pOutput = subprocess.Popen( config_storcli.cmd_detail, stdout=subprocess.PIPE )
|
||||
|
||||
fw = open( sys.argv[1], "w")
|
||||
transform.saveOutput( pOutput.stdout, fw)
|
||||
fw.close()
|
|
@ -0,0 +1,158 @@
|
|||
CLI Version = 007.0606.0000.0000 Mar 20, 2018
|
||||
Operating system = Linux 4.4.0-21-generic
|
||||
Controller = 0
|
||||
Status = Success
|
||||
Description = Show Diskgroup Succeeded
|
||||
|
||||
|
||||
TOPOLOGY :
|
||||
========
|
||||
|
||||
-----------------------------------------------------------------------------
|
||||
DG Arr Row EID:Slot DID Type State BT Size PDC PI SED DS3 FSpace TR
|
||||
-----------------------------------------------------------------------------
|
||||
0 - - - - Cac0 Optl N 372.093 GB dflt N N none N N
|
||||
0 0 - - - RAID0 Optl N 372.093 GB dflt N N none N N
|
||||
0 0 0 8:11 46 DRIVE Onln N 372.093 GB dflt N N none - N
|
||||
1 - - - - RAID6 Optl N 109.152 TB dflt N N dflt N N
|
||||
1 0 - - - RAID6 Optl N 109.152 TB dflt N N dflt N N
|
||||
1 0 0 9:0 42 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
1 0 1 9:1 14 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
1 0 2 9:2 18 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
1 0 3 9:3 38 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
1 0 4 9:4 41 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
1 0 5 9:5 23 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
1 0 6 9:6 26 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
1 0 7 9:7 21 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
1 0 8 9:8 13 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
1 0 9 9:9 15 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
1 0 10 9:10 10 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
1 0 11 9:11 43 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
1 0 12 9:12 27 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
1 0 13 9:13 24 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
1 0 14 9:14 44 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
1 0 15 9:15 28 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
1 0 16 9:16 19 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 - - - - RAID6 Optl N 109.152 TB dflt N N dflt N N
|
||||
2 0 - - - RAID6 Optl N 109.152 TB dflt N N dflt N N
|
||||
2 0 0 9:17 40 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 0 1 9:18 22 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 0 2 9:19 29 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 0 3 9:20 25 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 0 4 9:21 37 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 0 5 9:22 34 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 0 6 9:23 35 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 0 7 8:0 12 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 0 8 8:1 17 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 0 9 8:2 32 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 0 10 8:3 20 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 0 11 8:4 16 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 0 12 8:5 36 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 0 13 8:6 11 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 0 14 8:7 39 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 0 15 8:8 31 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
2 0 16 8:9 33 DRIVE Onln N 7.276 TB dflt N N dflt - N
|
||||
- - - 8:10 30 DRIVE GHS - 7.276 TB - - - - - N
|
||||
-----------------------------------------------------------------------------
|
||||
|
||||
DG=Disk Group Index|Arr=Array Index|Row=Row Index|EID=Enclosure Device ID
|
||||
DID=Device ID|Type=Drive Type|Onln=Online|Rbld=Rebuild|Dgrd=Degraded
|
||||
Pdgd=Partially degraded|Offln=Offline|BT=Background Task Active
|
||||
PDC=PD Cache|PI=Protection Info|SED=Self Encrypting Drive|Frgn=Foreign
|
||||
DS3=Dimmer Switch 3|dflt=Default|Msng=Missing|FSpace=Free Space Present
|
||||
TR=Transport Ready
|
||||
|
||||
|
||||
VD LIST :
|
||||
=======
|
||||
|
||||
---------------------------------------------------------------
|
||||
DG/VD TYPE State Access Consist Cache Cac sCC Size Name
|
||||
---------------------------------------------------------------
|
||||
0/0 Cac0 Optl RW Yes RWTD - ON 372.093 GB
|
||||
1/1 RAID6 Optl RW Yes RWBD - ON 109.152 TB
|
||||
2/2 RAID6 Optl RW Yes RWBD - ON 109.152 TB
|
||||
---------------------------------------------------------------
|
||||
|
||||
Cac=CacheCade|Rec=Recovery|OfLn=OffLine|Pdgd=Partially Degraded|Dgrd=Degraded
|
||||
Optl=Optimal|RO=Read Only|RW=Read Write|HD=Hidden|TRANS=TransportReady|B=Blocked|
|
||||
Consist=Consistent|R=Read Ahead Always|NR=No Read Ahead|WB=WriteBack|
|
||||
AWB=Always WriteBack|WT=WriteThrough|C=Cached IO|D=Direct IO|sCC=Scheduled
|
||||
Check Consistency
|
||||
|
||||
Total VD Count = 3
|
||||
|
||||
DG Drive LIST :
|
||||
=============
|
||||
|
||||
---------------------------------------------------------------------------------
|
||||
EID:Slt DID State DG Size Intf Med SED PI SeSz Model Sp Type
|
||||
---------------------------------------------------------------------------------
|
||||
8:11 46 Onln 0 372.093 GB SATA SSD N N 512B INTEL SSDSC2BA400G4 U -
|
||||
9:0 42 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:1 14 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:2 18 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:3 38 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:4 41 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:5 23 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:6 26 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:7 21 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:8 13 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:9 15 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:10 10 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:11 43 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:12 27 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:13 24 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:14 44 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:15 28 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:16 19 Onln 1 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:17 40 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:18 22 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:19 29 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:20 25 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:21 37 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:22 34 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
9:23 35 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
8:0 12 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
8:1 17 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
8:2 32 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
8:3 20 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
8:4 16 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
8:5 36 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
8:6 11 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
8:7 39 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
8:8 31 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
8:9 33 Onln 2 7.276 TB SAS HDD N N 512B HUH728080AL5200 U -
|
||||
---------------------------------------------------------------------------------
|
||||
|
||||
EID-Enclosure Device ID|Slt-Slot No.|DID-Device ID|DG-DriveGroup
|
||||
DHS-Dedicated Hot Spare|UGood-Unconfigured Good|GHS-Global Hotspare
|
||||
UBad-Unconfigured Bad|Onln-Online|Offln-Offline|Intf-Interface
|
||||
Med-Media Type|SED-Self Encryptive Drive|PI-Protection Info
|
||||
SeSz-Sector Size|Sp-Spun|U-Up|D-Down/PowerSave|T-Transition|F-Foreign
|
||||
UGUnsp-Unsupported|UGShld-UnConfigured shielded|HSPShld-Hotspare shielded
|
||||
CFShld-Configured shielded|Cpybck-CopyBack|CBShld-Copyback Shielded
|
||||
|
||||
Total Drive Count = 35
|
||||
|
||||
UN-CONFIGURED DRIVE LIST :
|
||||
========================
|
||||
|
||||
----------------------------------------------------------------------------
|
||||
EID:Slt DID State DG Size Intf Med SED PI SeSz Model Sp Type
|
||||
----------------------------------------------------------------------------
|
||||
8:10 30 GHS - 7.276 TB SAS HDD N N 512B HUH728080AL5200 D -
|
||||
----------------------------------------------------------------------------
|
||||
|
||||
EID-Enclosure Device ID|Slt-Slot No.|DID-Device ID|DG-DriveGroup
|
||||
DHS-Dedicated Hot Spare|UGood-Unconfigured Good|GHS-Global Hotspare
|
||||
UBad-Unconfigured Bad|Onln-Online|Offln-Offline|Intf-Interface
|
||||
Med-Media Type|SED-Self Encryptive Drive|PI-Protection Info
|
||||
SeSz-Sector Size|Sp-Spun|U-Up|D-Down/PowerSave|T-Transition|F-Foreign
|
||||
UGUnsp-Unsupported|UGShld-UnConfigured shielded|HSPShld-Hotspare shielded
|
||||
CFShld-Configured shielded|Cpybck-CopyBack|CBShld-Copyback Shielded
|
||||
|
||||
Unconfigured Drive Count = 1
|
||||
|
||||
|
||||
|
|
@ -3,7 +3,6 @@
|
|||
mailto="lopez@uci.edu"
|
||||
errorsto="lopez@uci.edu"
|
||||
scriptdir="/root/raid_check"
|
||||
# previous good run of storcli to compare against current output
|
||||
masterfile="$scriptdir/raid.master.txt"
|
||||
subject="`hostname` RAID check `date +%Y-%m-%d`"
|
||||
sendemail="n"
|
||||
|
|
|
@ -0,0 +1,29 @@
|
|||
import re
|
||||
|
||||
def saveOutput( readfh, writefh ):
|
||||
convertDriveStatus = False
|
||||
separatorCount = 0
|
||||
|
||||
isTopologyHeader = re.compile( "DG Arr Row EID:Slot DID Type State BT Size PDC PI SED DS3 FSpace TR " )
|
||||
isUnconfiguredHeader = re.compile( "EID:Slt DID State DG Size Intf Med SED PI SeSz Model Sp Type " )
|
||||
isSeparator = re.compile( "^-+$" )
|
||||
|
||||
for line in readfh.readlines():
|
||||
if convertDriveStatus:
|
||||
if isSeparator.match( line ):
|
||||
if separatorCount == 1:
|
||||
separatorCount = 0
|
||||
convertDriveStatus = False
|
||||
else:
|
||||
separatorCount += 1
|
||||
else:
|
||||
if line[36] == "Y":
|
||||
# set background task status to N for topology report
|
||||
line = line[0:36] + "N" + line[37:]
|
||||
elif line[68] == "U":
|
||||
# set drive spun status to D for un-configured drives report
|
||||
line = line[0:68] + "D" + line[69:]
|
||||
else:
|
||||
if isTopologyHeader.match( line ) or isUnconfiguredHeader.match( line ):
|
||||
convertDriveStatus = True
|
||||
writefh.write( line )
|
Loading…
Reference in New Issue