Munin contributed plugins

#!/bin/bash

######################################################################################################################

#                 Plugin to monitor NFS statistics of EMC VNX 5300 Unified Storage system                            #

######################################################################################################################

# Author: Evgeny Beysembaev <megabotva@gmail.com>

#####################################

#         Description               #

#####################################

# The plugin monitors NFS v3 statistics of EMC Unified Storage system. Probably it can also be compatible with 

# other Isilon or Celerra systems. It uses SSH to connect to Control Stations, then remotely executes 

# /nas/sbin/server_stats and fetches and parses data from it. It supports gathering data both from active/active 

# and active/passive Datamover configurations, ignoring offline or standby Datamovers. If all Datamovers are 

# offline or absent, the plugin returns error.

# This plugin also automatically chooses Primary Control Station from the list by calling /nasmcd/sbin/getreason and 

# /nasmcd/sbin/t2slot.

# 

# At the moment data is gathered from the following statistics sources:

# nfs.v3.op - Tons of timings about NFS RPC calls

# nfs.client - Here new Client addresses are rescanned and added automatically.

# 

# It's quite easy to comment out unneeded data to make graphs less overloaded or to add new statistics sources.

#####################################

#        Configuration              #

#####################################

# The plugin uses SSH to connect to Control Stations. It's possible to use 'nasadmin' user, but it would be better

# if you create read-only global user by Unisphere Client. The user should have only Operator role.

# I created "operator" user but due to the fact that Control Stations already had one internal "operator" user,

# the new one was called "operator1". So be careful.

# 

# On munin-node side choose a user which will be used to connect through SSH. Generally user "munin" is ok. Then,

# execute "sudo su munin -s /bin/bash", "ssh-keygen" and "ssh-copy-id" to both Control Stations with newly created 

# user.

# 

# Make a link from /usr/share/munin/plugins/emc_vnx_nfsv3_stats to /etc/munin/plugins/emc_vnx_nfsv3_stats_<NAME>,

# where <NAME> is any arbitrary name of your storage system. The plugin will return <NAME> in its answer 

# as "host_name" field.

# Assume your storage system is called "VNX5300".

# 

# Make a configuration file at /etc/munin/plugin-conf.d/emc_vnx_nfsv3_stats_VNX5300

# 

# [emc_vnx_nfsv3_stats_vnx5300]

# user munin							# SSH Client local user

# env.username operator1					# Remote user with Operator role

# env.cs_addr 192.168.1.1 192.168.1.2				# Control Stations addresses

# env.nas_servers server_2 server_3				# This is the default value and can be omitted

#####################################

#           History                 #

#####################################

# 08.11.2016 - First Release

######################################################################################################################

export LANG=C

TARGET=$(echo "${0##*/}" | cut -d _ -f 5)

: ${nas_servers:="server_2 server_3"}

SSH_CHECK='ssh -q $username@$CS "/nasmcd/sbin/getreason | grep -w slot_\`/nasmcd/sbin/t2slot\` | cut -d- -f1"'

if [ "$1" = "autoconf" ]; then

	echo "yes"

	exit 0

fi

if [ -z "$username" ]; then

	echo "No username!"

	exit 1

fi

if [ -z "$cs_addr" ]; then

	echo "No control station addresses!"

	exit 1

fi

#Choosing Cotrol Station. Code have to be "10"

for CS in $cs_addr; do

	if [ "`eval $SSH_CHECK`" -eq "10" ]; then

#		echo "$CS is Primary"

		PRIMARY_CS=$CS

		break

	fi

done

if [ -z "$PRIMARY_CS" ]; then

	echo "No alive primary Control Station from list \"$cs_addr\"";

	exit 1;

fi

SSH="ssh -q $username@$PRIMARY_CS . /home/operator1/.bash_profile; "

echo "host_name ${TARGET}"

if [ "$1" = "config" ] ; then

	for server in $nas_servers; do

		$SSH nas_server -i $server | grep -q 'type *= nas'

		if [ "$?" != 0 ]  ; then continue; fi

		nas_server_ok=TRUE

#nfs.v3.op data

		member_elements=`$SSH server_stats $server -info nfs.v3.op | grep member_elements | sed -ne 's/^.*= //p'`

		IFS=',' read -ra graphs <<< $member_elements

		echo "multigraph vnx_emc_calls_s

graph_title EMC VNX 5300 NFS V3 Calls per second

graph_vlabel Calls

graph_category nfs

graph_args --base 1000"

		for graph in "${graphs[@]}"; do

        	        field=`echo "$graph" | cut -d '.' -f4 `

			echo "${server}_$field.label $server $field"

        	done

		echo "

multigraph vnx_emc_usec_call

graph_title EMC VNX 5300 NFS V3 uSeconds per call

graph_vlabel uSec / call

graph_category nfs

graph_args --base 1000"

		for graph in "${graphs[@]}"; do

                	field=`echo "$graph" | cut -d '.' -f4 `

			echo "${server}_$field.label $server $field"

        	done

		echo "

multigraph vnx_emc_op_percent

graph_title EMC VNX 5300 NFS V3 Op %

graph_vlabel %

graph_category nfs"

		for graph in "${graphs[@]}"; do

                	field=`echo "$graph" | cut -d '.' -f4 `

			echo "${server}_$field.label $server $field"

			echo "${server}_$field.min 0"

        	done

#nfs.client data

#							 Total    Read     Write   Suspicious   Total    Read     Write      Avg   

#                                                      Ops/s    Ops/s    Ops/s    Ops diff    KiB/s    KiB/s    KiB/s   uSec/call

		member_elements=`$SSH server_stats server_2 -monitor nfs.client -count 1 -terminationsummary no -titles never | sed -ne 's/^.*id=//p' | cut -d' ' -f1`

		readarray graphs2 <<< $member_elements

		echo "

multigraph vnx_emc_nfs_client_ops_s

graph_title EMC VNX 5300 NFS Client Ops/s

graph_vlabel Ops/s

graph_category nfs"

		echo -n "graph_order "

		for graph in "${graphs2[@]}"; do

                        field=`echo "$graph" | sed -ne 's/\./_/pg' `

			echo -n "${server}_${field}_r ${server}_${field}_w ${server}_${field}_t ${server}_${field}_s "

		done

		echo " "

		for graph in "${graphs2[@]}"; do

                        field=`echo "$graph" | sed -ne 's/\./_/pg' `

                        echo "${server}_${field}_r.label $server $field Read Ops/s"

                        echo "${server}_${field}_w.label $server $field Write Ops/s"

                        echo "${server}_${field}_w.draw STACK"

                        echo "${server}_${field}_t.label $server $field Total Ops/s"

			echo "${server}_${field}_s.label $server $field Suspicious Ops diff"

                done

		echo "

multigraph vnx_emc_nfs_client_kib_s

graph_title EMC VNX 5300 NFS Client KiB/s

graph_vlabel KiB/s

graph_category nfs"

		echo -n "graph_order "

		for graph in "${graphs2[@]}"; do

                        field=`echo "$graph" | sed -ne 's/\./_/pg' `

			echo -n "${server}_${field}_r ${server}_${field}_w ${server}_${field}_t "

		done

		echo " "

		for graph in "${graphs2[@]}"; do

                        field=`echo "$graph" | sed -ne 's/\./_/pg' `

                        echo "${server}_${field}_r.label $server $field Read KiB/s"

                        echo "${server}_${field}_w.label $server $field Write KiB/s"

                        echo "${server}_${field}_w.draw STACK"

                        echo "${server}_${field}_t.label $server $field Total KiB/s"

                done

		echo "

multigraph vnx_emc_nfs_client_avg_usec

graph_title EMC VNX 5300 NFS Client Avg uSec/call

graph_vlabel uSec/call

graph_category nfs"

		for graph in "${graphs2[@]}"; do

                        field=`echo "$graph" | sed -ne 's/\./_/pg' `

                        echo "${server}_${field}.label $server $field Avg uSec/call"

                done

	done

	if [ -z $nas_server_ok ]; then

		echo "No active data movers!"

		exit 1

	fi

exit 0

fi

#nfs.v3.op data

for server in $nas_servers; do

	$SSH nas_server -i $server | grep -q 'type *= nas'

	if [ "$?" != 0 ]  ; then continue; fi

	nas_server_ok=TRUE

	member_elements=`$SSH server_stats $server -monitor nfs.v3.op -count 1 -terminationsummary no -titles never | sed -ne 's/^.*v3/v3/p'`

	NUMCOL=5

	LINES=`wc -l <<< $member_elements`

	while IFS=$'\n' read -ra graphs ; do

		element+=( $graphs )

	done <<< $member_elements

	echo "multigraph vnx_emc_calls_s"

	for ((i=0; i<$((LINES)); i++ )); do

		echo "${server}_${element[i*$NUMCOL]}".value "${element[i*$NUMCOL+1]}"

	done

	echo "

multigraph vnx_emc_usec_call"

	for ((i=0; i<$((LINES)); i++ )); do

		echo "${server}_${element[i*$NUMCOL]}".value "${element[i*$NUMCOL+3]}"

	done

	echo "

multigraph vnx_emc_op_percent"

	for ((i=0; i<$((LINES)); i++ )); do

		echo "${server}_${element[i*$NUMCOL]}".value "${element[i*$NUMCOL+4]}"

	done

	element=()

#nfs.client data

	echo "

multigraph vnx_emc_nfs_client_ops_s"

        member_elements=`$SSH server_stats server_2 -monitor nfs.client -count 1 -terminationsummary no -titles never | sed -ne 's/^.*id=//p'`

	NUMCOL=9

        LINES=`wc -l <<< $member_elements`

	while IFS=$'\n' read -ra graphs; do

		element+=($graphs)

	done  <<< $member_elements

	for (( i=0; i<$((LINES)); i++ )); do

		client=` echo ${element[i*$NUMCOL]} | sed -ne 's/\./_/pg'`

               echo "${server}_${client}_r".value "${element[$i*$NUMCOL+2]}"

               echo "${server}_${client}_w".value "${element[$i*$NUMCOL+3]}"

               echo "${server}_${client}_t".value "${element[$i*$NUMCOL+1]}"

               echo "${server}_${client}_s".value "${element[$i*$NUMCOL+4]}"

        done

	echo "

multigraph vnx_emc_nfs_client_kib_s"

	for (( i=0; i<$((LINES)); i++ )); do

		client=` echo ${element[i*$NUMCOL]} | sed -ne 's/\./_/pg'`

               echo "${server}_${client}_r".value "${element[$i*$NUMCOL+6]}"

               echo "${server}_${client}_w".value "${element[$i*$NUMCOL+7]}"

               echo "${server}_${client}_t".value "${element[$i*$NUMCOL+5]}"

        done

	echo "

multigraph vnx_emc_nfs_client_avg_usec"

	for (( i=0; i<$((LINES)); i++ )); do

		client=` echo ${element[i*$NUMCOL]} | sed -ne 's/\./_/pg'`

		echo "${server}_${client}".value "${element[$i*$NUMCOL+8]}"

	done

done

if [ -z $nas_server_ok ]; then

	echo "No active data movers!"

	exit 1

fi

exit 0