diff -r 000000000000 -r 8547a1ce445e lib/arborist/monitor/snmp.rb --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/arborist/monitor/snmp.rb Thu Jun 02 11:50:54 2016 -0700 @@ -0,0 +1,402 @@ +# -*- ruby -*- +# vim: set noet nosta sw=4 ts=4 : +#encoding: utf-8 +# +# SNMP checks for Arborist. Requires an SNMP agent to be installed +# on target machine, and the various "pieces" enabled. For your platform. +# +# For example, for disk monitoring with Net-SNMP, you'll want to set +# 'includeAllDisks' in the snmpd.conf. bsnmpd on FreeBSD benefits from +# the 'bsnmp-ucd' package. Etc. +# + +require 'loggability' +require 'arborist/monitor' unless defined?( Arborist::Monitor ) +require 'snmp' + +using Arborist::TimeRefinements + +# SNMP specific monitors and monitor logic. +# +class Arborist::Monitor::SNMP + extend Loggability + log_to :arborist + + # The version of this library. + VERSION = '0.1.0' + + # "Modes" that this monitor understands. + VALID_MODES = %i[ disk load memory swap process ] + + # The OID that returns the system environment. + IDENTIFICATION_OID = '1.3.6.1.2.1.1.1.0' + + # For net-snmp systems, ignore mount types that match + # this regular expression. This includes null/union mounts + # and NFS, currently. + STORAGE_IGNORE = %r{25.3.9.(?:2|14)$} + + # The OID that matches a local windows hard disk. Anything else + # is a remote (SMB) mount. + WINDOWS_DEVICE = '1.3.6.1.2.1.25.2.1.4' + + # OIDS required to pull disk information from net-snmp. + # + STORAGE_NET_SNMP = [ + '1.3.6.1.4.1.2021.9.1.2', # paths + '1.3.6.1.2.1.25.3.8.1.4', # types + '1.3.6.1.4.1.2021.9.1.9' # percents + ] + + # OIDS required to pull disk information from Windows. + # + STORAGE_WINDOWS = [ + '1.3.6.1.2.1.25.2.3.1.2', # types + '1.3.6.1.2.1.25.2.3.1.3', # paths + '1.3.6.1.2.1.25.2.3.1.5', # totalsize + '1.3.6.1.2.1.25.2.3.1.6' # usedsize + ] + + # OIDS for discovering memory usage. + # + MEMORY = { + swap_total: '1.3.6.1.4.1.2021.4.3.0', + swap_avail: '1.3.6.1.4.1.2021.4.4.0', + mem_avail: '1.3.6.1.4.1.2021.4.6.0' + } + + # OIDS for discovering system load. + # + LOAD = { + five_min: '1.3.6.1.4.1.2021.10.1.3.2' + } + + # OIDS for discovering running processes. + # + PROCESS = { + list: '1.3.6.1.2.1.25.4.2.1.4', + args: '1.3.6.1.2.1.25.4.2.1.5' + } + + + # Defaults for instances of this monitor + # + DEFAULT_OPTIONS = { + timeout: 2, + retries: 1, + community: 'public', + port: 161, + storage_error_at: 95, # in percent full + load_error_at: 7, + swap_error_at: 25, # in percent remaining + mem_error_at: 51200, # in kilobytes + processes: [] # list of procs to match + } + + + ### This monitor is complex enough to require creating an instance from the caller. + ### Provide a friendlier error message the class was provided to exec() directly. + ### + def self::run( nodes ) + self.log.error "Please use %s via an instance." % [ self.name ] + return {} + end + + + ### Create a new instance of this monitor. + ### + def initialize( options=DEFAULT_OPTIONS ) + options = DEFAULT_OPTIONS.merge( options || {} ) + + options.each do |name, value| + self.public_send( "#{name}=", value ) + end + end + + + # The mode (section) that this SMMP instance should check. + # Must be a +VALID_MODES+ mode. + attr_reader :mode + + # Mapping of node addresses back to the node identifier. + attr_reader :identifiers + + # The results from the SNMP daemons, keyed by address. + attr_reader :results + + # A timeout in seconds if the SNMP server isn't responding. + attr_accessor :timeout + + # Retry with the timeout this many times. Defaults to 1. + attr_accessor :retries + + # The SNMP UDP port, if running on non default. + attr_accessor :port + + # The community string to connect with. + attr_accessor :community + + # Set an error if mount points are above this percentage. + attr_accessor :storage_error_at + + # Set an error if the 5 minute load average exceeds this. + attr_accessor :load_error_at + + # Set an error if used swap exceeds this percentage. + attr_accessor :swap_error_at + + # Set an error if memory used is below this many kilobytes. + attr_accessor :mem_error_at + + # Set an error if processes in this array aren't running. + attr_accessor :processes + + + ### Set the SNMP mode, after validation. + ### + def mode=( mode ) + unless VALID_MODES.include?( mode.to_sym ) + self.log.error "Unknown SNMP mode: %s" % [ mode ] + return nil + end + + @mode = mode.to_sym + @results = {} + end + + + ### Perform the monitoring checks. + ### + def run( nodes ) + self.log.debug "Got nodes to SNMP check: %p" % [ nodes ] + + # Sanity check. + # + unless self.mode + self.log.error "You must set the 'mode' for the SNMP monitor. (%s)" % [ VALID_MODES.join( ', ' ) ] + return {} + end + + # Create mapping of addresses back to node identifiers. + # + @identifiers = nodes.each_with_object({}) do |(identifier, props), hash| + next unless props.key?( 'addresses' ) + address = props[ 'addresses' ].first + hash[ address ] = identifier + end + + # Perform the work! + # + threads = [] + self.identifiers.keys.each do |host| + thr = Thread.new do + Thread.current.abort_on_exception = true + opts = { + host: host, + port: self.port, + community: self.community, + timeout: self.timeout, + retries: self.retries + } + + begin + SNMP::Manager.open( opts ) do |snmp| + case self.mode + when :disk + self.gather_disks( snmp, host ) + when :load + self.gather_load( snmp, host ) + when :memory + self.gather_free_memory( snmp, host ) + when :swap + self.gather_swap( snmp, host ) + when :process + self.gather_processlist( snmp, host ) + end + end + rescue SNMP::RequestTimeout + self.results[ host ] = { + error: "Host is not responding to SNMP requests." + } + rescue StandardError => err + self.results[ host ] = { + error: "Network is not accessible. (%s: %s)" % [ err.class.name, err.message ] + } + end + end + threads << thr + end + + # Wait for thread completion + threads.map( &:join ) + + # Map everything back to identifier -> attribute(s), and send to the manager. + # + reply = self.results.each_with_object({}) do |(address, results), hash| + identifier = self.identifiers[ address ] or next + hash[ identifier ] = results + end + self.log.debug "Sending to manager: %p" % [ reply ] + return reply + end + + + ######### + protected + ######### + + ### Collect the load information for +host+ from an existing + ### (and open) +snmp+ connection. + ### + def gather_load( snmp, host ) + self.log.debug "Getting system load for: %s" % [ host ] + load5 = snmp.get( SNMP::ObjectId.new( LOAD[:five_min] ) ).varbind_list.first.value.to_f + self.log.debug " Load on %s: %0.2f" % [ host, load5 ] + + if load5 >= self.load_error_at + self.results[ host ] = { + error: "Load has exceeded %0.2f over a 5 minute average" % [ self.load_error_at ], + load5: load5 + } + else + self.results[ host ] = { load5: load5 } + end + end + + + ### Collect available memory information for +host+ from an existing + ### (and open) +snmp+ connection. + ### + def gather_free_memory( snmp, host ) + self.log.debug "Getting available memory for: %s" % [ host ] + mem_avail = snmp.get( SNMP::ObjectId.new( MEMORY[:mem_avail] ) ).varbind_list.first.value.to_f + self.log.debug " Available memory on %s: %0.2f" % [ host, mem_avail ] + + if mem_avail <= self.mem_error_at + self.results[ host ] = { + error: "Available memory is under %0.1fMB" % [ self.mem_error_at.to_f / 1024 ], + available_memory: mem_avail + } + else + self.results[ host ] = { available_memory: mem_avail } + end + end + + + ### Collect used swap information for +host+ from an existing (and + ### open) +snmp+ connection. + ### + def gather_swap( snmp, host ) + self.log.debug "Getting used swap for: %s" % [ host ] + + swap_total = snmp.get( SNMP::ObjectId.new(MEMORY[:swap_total]) ).varbind_list.first.value.to_f + swap_avail = snmp.get( SNMP::ObjectId.new(MEMORY[:swap_avail]) ).varbind_list.first.value.to_f + swap_used = ( "%0.2f" % ((swap_avail / swap_total.to_f * 100 ) - 100).abs ).to_f + self.log.debug " Swap in use on %s: %0.2f" % [ host, swap_used ] + + if swap_used >= self.swap_error_at + self.results[ host ] = { + error: "%0.2f%% swap in use" % [ swap_used ], + swap_used: swap_used + } + else + self.results[ host ] = { swap_used: swap_used } + end + end + + + ### Collect mount point usage for +host+ from an existing (and open) + #### +snmp+ connection. + ### + def gather_disks( snmp, host ) + self.log.debug "Getting disk information for %s" % [ host ] + errors = [] + results = {} + mounts = self.get_disk_percentages( snmp ) + + mounts.each_pair do |path, percentage| + if percentage >= self.storage_error_at + errors << "Mount %s at %d%% capacity" % [ path, percentage ] + end + end + + results[ :mounts ] = mounts + results[ :error ] = errors.join( ', ' ) unless errors.empty? + + self.results[ host ] = results + end + + + ### Collect running processes on +host+ from an existing (and open) + #### +snmp+ connection. + ### + def gather_processlist( snmp, host ) + self.log.debug "Getting running process list for %s" % [ host ] + procs = [] + + snmp.walk([ PROCESS[:list], PROCESS[:args] ]) do |list| + process = list[0].value.to_s + args = list[1].value.to_s + procs << "%s %s " % [ process, args ] + end + + # Check against the running stuff, setting an error if + # one isn't found. + # + errors = [] + Array( self.processes ).each do |process| + process_r = Regexp.new( process ) + found = procs.find{|p| p.match(process_r) } + errors << "Process '%s' is not running" % [ process, host ] unless found + end + + self.log.debug " %d running processes" % [ procs.length ] + if errors.empty? + self.results[ host ] = {} + else + self.results[ host ] = { error: errors.join( ', ' ) } + end + end + + + ### Given a SNMP object, return a hash of: + ### + ### device path => percentage full + ### + def get_disk_percentages( snmp ) + + # Does this look like a windows system, or a net-snmp based one? + system_type = snmp.get( SNMP::ObjectId.new( IDENTIFICATION_OID ) ).varbind_list.first.value + disks = {} + + # Windows has it's own MIBs. + # + if system_type =~ /windows/i + snmp.walk( STORAGE_WINDOWS ) do |list| + next unless list[0].value.to_s == WINDOWS_DEVICE + disks[ list[1].value.to_s ] = ( list[3].value.to_f / list[2].value.to_f ) * 100 + end + return disks + end + + # Everything else. + # + snmp.walk( STORAGE_NET_SNMP ) do |list| + mount = list[0].value.to_s + next if mount == 'noSuchInstance' + + next if list[2].value.to_s == 'noSuchInstance' + used = list[2].value.to_i + + typeoid = list[1].value.join('.').to_s + next if typeoid =~ STORAGE_IGNORE + next if mount =~ /\/(?:dev|proc)$/ + + self.log.debug " %s -> %s -> %s" % [ mount, typeoid, used ] + disks[ mount ] = used + end + + return disks + end +end # class Arborist::Monitor::SNMP +