Oct 202011
 

We had performance issues on our NFS servers, very poor performances.
after analyze we have seen that the bottleneck came from the disks, as always;
but tuning the FS did not solve the problem.

I talk about 1500 NFS mountpoint (running on 9 nodes from ServiceGuard cluster solution) as in my actual mission, everything works through NFS.
The homedir of course but also the applications, the logs, libraries etc…

so, i’ve wrote a script to obtain the fragmentation state of all VxFS filesystem on HP-UX.
this script runs every weeks, send a mail with a high importance when fragmentation is detected, and indicates what to do to reduce the fragmentation.

here is the script:

Download : fragmentation_vxfs_en1.sh

#! /bin/ksh
#
# vim:tabstop=3:syntax=sh:
#
# ---------
## Script  : fragmentation_vxfs.sh
## Objet   : envoie un rapport de fragmentation VXFS si besoin
##----------------------------------------------------------------------------
# 0.1 31-jul-09 - Cedrick Gaillard
# 1.1 01-sep-09 - plus d'analyse sur /tmp
#               - plus d'analyse sur les FS utilises a 5% ou moins
#               - cree le repertoire $MOUNTPOINT/lost+found pour le lock fsadm
#                 si besoin.
#               - les commandes a lancer pour defragmenter accompagnent
#                 maintenant les rapports.
#               - si plusieurs FS sont a defragmenter, une liste avec toutes 
#                 les commandes a lancer est affichee en premier, suivi du
#                 detail.
#               - on peut maintenant s'envoyer un mail de rapport au lieu de
#                 l'envoyer a tout le monde.
#               - reecrit plus proprement.
# 1.2 16-sep-09 - utilise bdfa si bdf.pl sort en erreur
#               - un seul bdf.pl/bdfa est lance maintenant
#               - n'analyse plus les FS utilises <= 15% d'utilisation
#               - leger changement sur l'output.
#               - n'analyse plus /tmp
# 1.5 24-sep-09 - gestion de seuils warnings et critiques.
#               - reecriture d'un partie du script awk.
# 1.6 20-oct-09 - on peut specifier des FS a analyser au lieu de tout analyser
# 1.7 03-nov-09 - correction du test de selection de l'importance des mails
#               - correction du test du % d'utilisation du FS.
# 1.8 12-avr-10 - affiche l'etat de fragmentation mme quand le FS n'est pas
#                 fragmente.
#               - legere modification du message de resume.
# 1.9 30-avr-10 - ajout de quelques commentaires.
#               - n'affiche plus que les fs a defragmenter dans le resume des
#                 commande a lancer.
#               - n'affiche plus les rapports, seulement les status.
#
#-----------------------------------------------------------------------------

SERVER_NAME=$(uname -n)
DEFRAG_COMMAND="/usr/sbin/fsadm -F vxfs -DEde -s"

TMPFILE="/var/tmp/${0##*/}.VARS.$$"

EXPEDITEUR="FRAGMENTATION REPORT<root@$(hostname)>"
DESTINATAIRE="Operator<operator@example.com>"
IMPORTANCE="Low"

# Keep it to '0', It will be automatically set to '1' if a FS needs to be defragmented.
SEND=0

# css decoration
CSS='<style type="text/css">
	<!--
	body { font-family:"Courier New",Courier,sans-serif; font-size:8pt; }
	h1   { font-size:12pt; color:darkblue; }
	.Rouge { color:#FF0000; font-weight: bold }
	.Orang { color:#FF8C00; font-weight: bold }
	.Verte { color:#00CC00; }
	-->
	</style>
'

# arguments
while [ "$#" -ne "0" ]
do
	case $1
	in
		-mail) [ -n "$2" ] && {
					DESTINATAIRE="$2"
					shift 1
				}
				shift 1
				;;
		-v)	# Permits to show status event if there is no fragmentation.
				VERBOSE=true
				shift 1
				;;
		*)		FS_TO_ANALYZE="$FS_TO_ANALYZE $1"
				shift 1
				;;
	esac
done

[ -x "/usr/bin/perl" ] && BDF="$(/usr/local/bin/bdf.pl -t vxfs 2>/dev/null |grep -v interpreter)"
[ "$?" = "0" ] || BDF="$(/usr/local/bin/bdfa -t vxfs 2>/dev/null)"
[ "$?" = "0" ] || BDF=
BDF_HEAD="$(echo "${BDF}" |head -1)"

if [ -n "$FS_TO_ANALYZE" ]
	then
		MOUNTPOINTS="$(bdf $FS_TO_ANALYZE |tail +2 |awk '{print $NF}')"
	else
		MOUNTPOINTS="$(grep " vxfs " /etc/mnttab |cut -d" " -f 2)"
fi

# grep " vxfs " /etc/mnttab |cut -d" " -f 2 |while read MOUNTPOINT
for MOUNTPOINT in $MOUNTPOINTS
do
	[ -d "${MOUNTPOINT}/lost+found" ] || mkdir "${MOUNTPOINT}/lost+found"
	chown 0:0 "${MOUNTPOINT}/lost+found"
	chmod 0700 "${MOUNTPOINT}/lost+found"
	[ "$MOUNTPOINT" = "/tmp" ] && continue

	[ "$BDF" != "" ] && {
	# We do not analyze FS used less than 15%
			[ "$(echo "$BDF" |awk '{print $5 " " $10}' | grep " ${MOUNTPOINT}$" |cut -d'%' -f1)" -le "15" ] && continue
	}


	# raw state of the fragmentation
	FRAGMENTATION="$(/usr/sbin/fsadm -F vxfs -E $MOUNTPOINT)"
	[ "$?" = "0" ] || {
		echo "ERROR: wannot analyze the mountpoint: $MOUNTPOINT"
		continue
	}

	# stack states by category
	unset ACTUAL_SMALL64 ACTUAL_SMALL8 ACTUAL_BLOCKS64
	ACTUAL_SMALL64="$( echo "${FRAGMENTATION}" |fgrep "% Free blocks in extents smaller than 64 blks:" |awk '{print $NF}')"
	ACTUAL_SMALL8="$(  echo "${FRAGMENTATION}" |fgrep "% Free blocks in extents smaller than  8 blks:" |awk '{print $NF}')"
	ACTUAL_BLOCKS64="$(echo "${FRAGMENTATION}" |fgrep "% blks allocated to extents 64 blks or larger:" |awk '{print $NF}')"

	[ -z "$ACTUAL_SMALL64" -o -z "$ACTUAL_SMALL8" -o -z "$ACTUAL_BLOCKS64" ] && {
		echo "ERROR: at least one of the values is empty concerning the mountpoint: $MOUNTPOINT"
		continue
	}

	STATUS=$(echo $ACTUAL_SMALL64 $ACTUAL_SMALL8 $ACTUAL_BLOCKS64 |awk 'BEGIN { RVAL=0 } {
		CRIT_SMALL64=70  ; WARN_SMALL64=50 ;
		CRIT_SMALL8=15   ; WARN_SMALL8=5 ;
		CRIT_BLOCKS64=4 ; WARN_BLOCKS64=6 ;

		ROUGE = "<span class=\"Rouge\">CRIT" ;
		ORANG = "<span class=\"Orang\">WARN" ;
		VERTE = "<span class=\"Verte\">GOOD" ;

		MSG = sprintf("--> %% Free blocks in extents smaller than 64 blks : %5.2f%%</span>", $1) ;
		SEUIL = sprintf("<span>- WARN if &gt;%s%% / CRIT if &gt;%s%%</span>", WARN_SMALL64, CRIT_SMALL64) ;
		if( $1 > CRIT_SMALL64 ) {
			printf("%s %s %s\n", ROUGE, MSG, SEUIL ) ; RVAL=2 ;
		} else if ( $1 > WARN_SMALL64 ) {
			printf("%s %s %s\n", ORANG, MSG, SEUIL ) ; RVAL=1 ;
		} else {
			printf("%s %s\n", VERTE, MSG ) ;
		}

		MSG = sprintf("--> %% Free blocks in extents smaller than  8 blks : %5.2f%%</span>", $2) ;
		SEUIL = sprintf("<span>- WARN if &gt;%s%% / CRIT IF &gt;%s%%</span>", WARN_SMALL8, CRIT_SMALL8) ;
		if( $2 > CRIT_SMALL8 ) {
			printf("%s %s %s\n", ROUGE, MSG, SEUIL ) ; RVAL = 2 ;
		} else if( $2 > WARN_SMALL8 ) {
			printf("%s %s %s\n", ORANG, MSG, SEUIL ) ; RVAL = 1 ;
		} else {
			printf("%s %s\n", VERTE, MSG ) ;
		}

		MSG = sprintf("--> %% blks allocated to extents 64 blks or larger : %5.2f%%</span>", $3) ;
		SEUIL = sprintf("<span>- WARN if &lt;%s%% / CRIT if &lt;%s%%</span>", WARN_BLOCKS64, CRIT_BLOCKS64) ;
		if( $3 < CRIT_BLOCKS64 ) {
			printf("%s %s %s\n", ROUGE, MSG, SEUIL ) ; RVAL = 2 ;
		} else if( $3 < WARN_BLOCKS64 ) {
			printf("%s %s %s\n", ORANG, MSG, SEUIL ) ; RVAL = 1 ;
		} else {
			printf("%s %s\n", VERTE, MSG ) ;
		}
	} END { exit RVAL ; }')

	RVAL=$?
	&#91; "$RVAL" -ne "0" &#93; && {
		SEND=1
		if &#91; "$RVAL" -eq "1" -a "$IMPORTANCE" != "High" &#93;
			then
				IMPORTANCE="Normal"
			else
				IMPORTANCE="High"
		fi
	}

	if &#91; "$BDF" != "" &#93;
		then
			REPORT_BDF="${BDF_HEAD}<br/>$(echo "${BDF}" |awk -v M=${MOUNTPOINT} '{if($NF == M){print $0; break}else{next;}}')"
		else
			REPORT_BDF=
	fi

	CONTENT="${CONTENT}
		<hr/>
		<h1>Fragmentation information on the mountpoint:</h1>
		<pre>${REPORT_BDF}</pre>"
	[ "$RVAL" -ne "0" -o "$VERBOSE" = "true" ] && {
		CONTENT="${CONTENT}<pre>$STATUS</pre>"
	}
#	[ "$RVAL" -ne "0" ] && {
#		CONTENT="${CONTENT}
#			<p>lancer la commande suivante pour d�fragmenter si besoin:<br/>
#			${DEFRAG_COMMAND} $MOUNTPOINT
#			</p>"
#	}
#	[ "$RVAL" -ne "0" -o $VERBOSE = "true" ] && {
#		CONTENT="${CONTENT}
#			<h2>STATUS:</h2>
#			<pre>$STATUS</pre>"
#	}
#	[ "$RVAL" -ne "0" ] && {
#		CONTENT="${CONTENT}
#			<h2>RAPPORT:</h2>
#			<pre>$FRAGMENTATION</pre>"
#	}
done

# En tete
echo "From: ${EXPEDITEUR}
To: ${DESTINATAIRE}
Importance: ${IMPORTANCE}
Subject:[${SERVER_NAME}] Fragmentation report
Content-Type: text/html
<html><head>${CSS}</head><body>" > $TMPFILE

if [ "$SEND" = "1" -o "$VERBOSE" = "true" ]
	then
		# If more than one FS, commands on the header
		[ "$(echo "${CONTENT}" |grep -c "${DEFRAG_COMMAND} ")" -gt "1" ] && {
		echo "<h1> INFO: Fragmentation detected on more than one FS</h1><p>please run the following commands to optimize files access<ul>" >> $TMPFILE
			echo "${CONTENT}" |grep "${DEFRAG_COMMAND} " |sed -e "s/	//g" -e "s/^/<li>/" -e "s/$/<\/li>/" >> $TMPFILE
			echo "</ul></p>" >> $TMPFILE
		}

		FS_TO_ANALYZE=$(echo "${CONTENT}" |grep "${DEFRAG_COMMAND} " |awk '{printf $NF " "}')

		echo "<p>The following command will permit you to verify the fragmentation state: <br/>
				$0 -v -mail $DESTINATAIRE $FS_TO_ANALYZE</p>
				<p>d&eacute;tails:</p>" >> $TMPFILE

		echo "${CONTENT}" >> $TMPFILE
	else
		/bin/rm -f $TMPFILE
		exit 0
fi

# footer
echo "<p>Script: $0</p> </body></html>" >> $TMPFILE

cat $TMPFILE |sendmail -t

/bin/rm -f $TMPFILE

and an example of the output:

 Laisser un commentaire

Vous pouvez utiliser ces tags et attributs HTML: <a href="" title=""> <abbr title=""> <acronym title=""> <b> <blockquote cite=""> <cite> <code> <del datetime=""> <em> <i> <q cite=""> <s> <strike> <strong>

(requis)

(requis)

Ce site utilise Akismet pour réduire les indésirables. En savoir plus sur comment les données de vos commentaires sont utilisées.