Oct 202011
We had performance issues on our NFS servers, very poor performances.
after analyze we have seen that the bottleneck came from the disks, as always;
but tuning the FS did not solve the problem.
I talk about 1500 NFS mountpoint (running on 9 nodes from ServiceGuard cluster solution) as in my actual mission, everything works through NFS.
The homedir of course but also the applications, the logs, libraries etc…
so, i’ve wrote a script to obtain the fragmentation state of all VxFS filesystem on HP-UX.
this script runs every weeks, send a mail with a high importance when fragmentation is detected, and indicates what to do to reduce the fragmentation.
here is the script:
Download : fragmentation_vxfs_en1.sh
#! /bin/ksh
#
# vim:tabstop=3:syntax=sh:
#
# ---------
## Script : fragmentation_vxfs.sh
## Objet : envoie un rapport de fragmentation VXFS si besoin
##----------------------------------------------------------------------------
# 0.1 31-jul-09 - Cedrick Gaillard
# 1.1 01-sep-09 - plus d'analyse sur /tmp
# - plus d'analyse sur les FS utilises a 5% ou moins
# - cree le repertoire $MOUNTPOINT/lost+found pour le lock fsadm
# si besoin.
# - les commandes a lancer pour defragmenter accompagnent
# maintenant les rapports.
# - si plusieurs FS sont a defragmenter, une liste avec toutes
# les commandes a lancer est affichee en premier, suivi du
# detail.
# - on peut maintenant s'envoyer un mail de rapport au lieu de
# l'envoyer a tout le monde.
# - reecrit plus proprement.
# 1.2 16-sep-09 - utilise bdfa si bdf.pl sort en erreur
# - un seul bdf.pl/bdfa est lance maintenant
# - n'analyse plus les FS utilises <= 15% d'utilisation
# - leger changement sur l'output.
# - n'analyse plus /tmp
# 1.5 24-sep-09 - gestion de seuils warnings et critiques.
# - reecriture d'un partie du script awk.
# 1.6 20-oct-09 - on peut specifier des FS a analyser au lieu de tout analyser
# 1.7 03-nov-09 - correction du test de selection de l'importance des mails
# - correction du test du % d'utilisation du FS.
# 1.8 12-avr-10 - affiche l'etat de fragmentation mme quand le FS n'est pas
# fragmente.
# - legere modification du message de resume.
# 1.9 30-avr-10 - ajout de quelques commentaires.
# - n'affiche plus que les fs a defragmenter dans le resume des
# commande a lancer.
# - n'affiche plus les rapports, seulement les status.
#
#-----------------------------------------------------------------------------
SERVER_NAME=$(uname -n)
DEFRAG_COMMAND="/usr/sbin/fsadm -F vxfs -DEde -s"
TMPFILE="/var/tmp/${0##*/}.VARS.$$"
EXPEDITEUR="FRAGMENTATION REPORT<root@$(hostname)>"
DESTINATAIRE="Operator<operator@example.com>"
IMPORTANCE="Low"
# Keep it to '0', It will be automatically set to '1' if a FS needs to be defragmented.
SEND=0
# css decoration
CSS='<style type="text/css">
<!--
body { font-family:"Courier New",Courier,sans-serif; font-size:8pt; }
h1 { font-size:12pt; color:darkblue; }
.Rouge { color:#FF0000; font-weight: bold }
.Orang { color:#FF8C00; font-weight: bold }
.Verte { color:#00CC00; }
-->
</style>
'
# arguments
while [ "$#" -ne "0" ]
do
case $1
in
-mail) [ -n "$2" ] && {
DESTINATAIRE="$2"
shift 1
}
shift 1
;;
-v) # Permits to show status event if there is no fragmentation.
VERBOSE=true
shift 1
;;
*) FS_TO_ANALYZE="$FS_TO_ANALYZE $1"
shift 1
;;
esac
done
[ -x "/usr/bin/perl" ] && BDF="$(/usr/local/bin/bdf.pl -t vxfs 2>/dev/null |grep -v interpreter)"
[ "$?" = "0" ] || BDF="$(/usr/local/bin/bdfa -t vxfs 2>/dev/null)"
[ "$?" = "0" ] || BDF=
BDF_HEAD="$(echo "${BDF}" |head -1)"
if [ -n "$FS_TO_ANALYZE" ]
then
MOUNTPOINTS="$(bdf $FS_TO_ANALYZE |tail +2 |awk '{print $NF}')"
else
MOUNTPOINTS="$(grep " vxfs " /etc/mnttab |cut -d" " -f 2)"
fi
# grep " vxfs " /etc/mnttab |cut -d" " -f 2 |while read MOUNTPOINT
for MOUNTPOINT in $MOUNTPOINTS
do
[ -d "${MOUNTPOINT}/lost+found" ] || mkdir "${MOUNTPOINT}/lost+found"
chown 0:0 "${MOUNTPOINT}/lost+found"
chmod 0700 "${MOUNTPOINT}/lost+found"
[ "$MOUNTPOINT" = "/tmp" ] && continue
[ "$BDF" != "" ] && {
# We do not analyze FS used less than 15%
[ "$(echo "$BDF" |awk '{print $5 " " $10}' | grep " ${MOUNTPOINT}$" |cut -d'%' -f1)" -le "15" ] && continue
}
# raw state of the fragmentation
FRAGMENTATION="$(/usr/sbin/fsadm -F vxfs -E $MOUNTPOINT)"
[ "$?" = "0" ] || {
echo "ERROR: wannot analyze the mountpoint: $MOUNTPOINT"
continue
}
# stack states by category
unset ACTUAL_SMALL64 ACTUAL_SMALL8 ACTUAL_BLOCKS64
ACTUAL_SMALL64="$( echo "${FRAGMENTATION}" |fgrep "% Free blocks in extents smaller than 64 blks:" |awk '{print $NF}')"
ACTUAL_SMALL8="$( echo "${FRAGMENTATION}" |fgrep "% Free blocks in extents smaller than 8 blks:" |awk '{print $NF}')"
ACTUAL_BLOCKS64="$(echo "${FRAGMENTATION}" |fgrep "% blks allocated to extents 64 blks or larger:" |awk '{print $NF}')"
[ -z "$ACTUAL_SMALL64" -o -z "$ACTUAL_SMALL8" -o -z "$ACTUAL_BLOCKS64" ] && {
echo "ERROR: at least one of the values is empty concerning the mountpoint: $MOUNTPOINT"
continue
}
STATUS=$(echo $ACTUAL_SMALL64 $ACTUAL_SMALL8 $ACTUAL_BLOCKS64 |awk 'BEGIN { RVAL=0 } {
CRIT_SMALL64=70 ; WARN_SMALL64=50 ;
CRIT_SMALL8=15 ; WARN_SMALL8=5 ;
CRIT_BLOCKS64=4 ; WARN_BLOCKS64=6 ;
ROUGE = "<span class=\"Rouge\">CRIT" ;
ORANG = "<span class=\"Orang\">WARN" ;
VERTE = "<span class=\"Verte\">GOOD" ;
MSG = sprintf("--> %% Free blocks in extents smaller than 64 blks : %5.2f%%</span>", $1) ;
SEUIL = sprintf("<span>- WARN if >%s%% / CRIT if >%s%%</span>", WARN_SMALL64, CRIT_SMALL64) ;
if( $1 > CRIT_SMALL64 ) {
printf("%s %s %s\n", ROUGE, MSG, SEUIL ) ; RVAL=2 ;
} else if ( $1 > WARN_SMALL64 ) {
printf("%s %s %s\n", ORANG, MSG, SEUIL ) ; RVAL=1 ;
} else {
printf("%s %s\n", VERTE, MSG ) ;
}
MSG = sprintf("--> %% Free blocks in extents smaller than 8 blks : %5.2f%%</span>", $2) ;
SEUIL = sprintf("<span>- WARN if >%s%% / CRIT IF >%s%%</span>", WARN_SMALL8, CRIT_SMALL8) ;
if( $2 > CRIT_SMALL8 ) {
printf("%s %s %s\n", ROUGE, MSG, SEUIL ) ; RVAL = 2 ;
} else if( $2 > WARN_SMALL8 ) {
printf("%s %s %s\n", ORANG, MSG, SEUIL ) ; RVAL = 1 ;
} else {
printf("%s %s\n", VERTE, MSG ) ;
}
MSG = sprintf("--> %% blks allocated to extents 64 blks or larger : %5.2f%%</span>", $3) ;
SEUIL = sprintf("<span>- WARN if <%s%% / CRIT if <%s%%</span>", WARN_BLOCKS64, CRIT_BLOCKS64) ;
if( $3 < CRIT_BLOCKS64 ) {
printf("%s %s %s\n", ROUGE, MSG, SEUIL ) ; RVAL = 2 ;
} else if( $3 < WARN_BLOCKS64 ) {
printf("%s %s %s\n", ORANG, MSG, SEUIL ) ; RVAL = 1 ;
} else {
printf("%s %s\n", VERTE, MSG ) ;
}
} END { exit RVAL ; }')
RVAL=$?
[ "$RVAL" -ne "0" ] && {
SEND=1
if [ "$RVAL" -eq "1" -a "$IMPORTANCE" != "High" ]
then
IMPORTANCE="Normal"
else
IMPORTANCE="High"
fi
}
if [ "$BDF" != "" ]
then
REPORT_BDF="${BDF_HEAD}<br/>$(echo "${BDF}" |awk -v M=${MOUNTPOINT} '{if($NF == M){print $0; break}else{next;}}')"
else
REPORT_BDF=
fi
CONTENT="${CONTENT}
<hr/>
<h1>Fragmentation information on the mountpoint:</h1>
<pre>${REPORT_BDF}</pre>"
[ "$RVAL" -ne "0" -o "$VERBOSE" = "true" ] && {
CONTENT="${CONTENT}<pre>$STATUS</pre>"
}
# [ "$RVAL" -ne "0" ] && {
# CONTENT="${CONTENT}
# <p>lancer la commande suivante pour d�fragmenter si besoin:<br/>
# ${DEFRAG_COMMAND} $MOUNTPOINT
# </p>"
# }
# [ "$RVAL" -ne "0" -o $VERBOSE = "true" ] && {
# CONTENT="${CONTENT}
# <h2>STATUS:</h2>
# <pre>$STATUS</pre>"
# }
# [ "$RVAL" -ne "0" ] && {
# CONTENT="${CONTENT}
# <h2>RAPPORT:</h2>
# <pre>$FRAGMENTATION</pre>"
# }
done
# En tete
echo "From: ${EXPEDITEUR}
To: ${DESTINATAIRE}
Importance: ${IMPORTANCE}
Subject:[${SERVER_NAME}] Fragmentation report
Content-Type: text/html
<html><head>${CSS}</head><body>" > $TMPFILE
if [ "$SEND" = "1" -o "$VERBOSE" = "true" ]
then
# If more than one FS, commands on the header
[ "$(echo "${CONTENT}" |grep -c "${DEFRAG_COMMAND} ")" -gt "1" ] && {
echo "<h1> INFO: Fragmentation detected on more than one FS</h1><p>please run the following commands to optimize files access<ul>" >> $TMPFILE
echo "${CONTENT}" |grep "${DEFRAG_COMMAND} " |sed -e "s/ //g" -e "s/^/<li>/" -e "s/$/<\/li>/" >> $TMPFILE
echo "</ul></p>" >> $TMPFILE
}
FS_TO_ANALYZE=$(echo "${CONTENT}" |grep "${DEFRAG_COMMAND} " |awk '{printf $NF " "}')
echo "<p>The following command will permit you to verify the fragmentation state: <br/>
$0 -v -mail $DESTINATAIRE $FS_TO_ANALYZE</p>
<p>détails:</p>" >> $TMPFILE
echo "${CONTENT}" >> $TMPFILE
else
/bin/rm -f $TMPFILE
exit 0
fi
# footer
echo "<p>Script: $0</p> </body></html>" >> $TMPFILE
cat $TMPFILE |sendmail -t
/bin/rm -f $TMPFILE
and an example of the output:
