Initial commit -- ssh base classes, converted from GroundControl and
authorMahlon E. Smith <mahlon@martini.nu>
Sat, 10 May 2014 17:40:36 -0700
changeset 0 aef8f9f4a788
child 1 8fb93d4764de
Initial commit -- ssh base classes, converted from GroundControl and re-worked for Symphony.
.hgignore
.pryrc
.rspec
.rvm.gems
.rvmrc
.simplecov
README.rdoc
Rakefile
lib/symphony/tasks/ssh.rb
lib/symphony/tasks/sshscript.rb
spec/helpers.rb
spec/symphony/tasks/.placeholder
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/.hgignore	Sat May 10 17:40:36 2014 -0700
@@ -0,0 +1,10 @@
+\.orig$
+\.rej$
+etc/.*\.(conf|yml)$
+\.DS_Store
+~$
+pkg/
+^ChangeLog$
+^docs/
+^coverage/
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/.pryrc	Sat May 10 17:40:36 2014 -0700
@@ -0,0 +1,14 @@
+#!/usr/bin/ruby -*- ruby -*-
+
+require 'pathname'
+
+begin
+	$LOAD_PATH.unshift( Pathname(__FILE__).dirname + 'lib' )
+	require 'symphony'
+	require 'symphony/metronome'
+
+rescue => e
+	$stderr.puts "Ack! Libraries failed to load: #{e.message}\n\t" +
+		e.backtrace.join( "\n\t" )
+end
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/.rspec	Sat May 10 17:40:36 2014 -0700
@@ -0,0 +1,1 @@
+-fd -c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/.rvm.gems	Sat May 10 17:40:36 2014 -0700
@@ -0,0 +1,12 @@
+inversion -v0.12.3
+net-sftp -v2.1.2
+net-ssh -v2.9.0
+rake -v0.9.6
+rdoc -v4.0.0
+rdoc-generator-fivefish -v0.1.0
+rspec -v3.0.0.beta2
+rspec-core -v3.0.0.beta2
+rspec-expectations -v3.0.0.beta2
+rspec-mocks -v3.0.0.beta2
+rspec-support -v3.0.0.beta2
+symphony -v0.6.0
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/.rvmrc	Sat May 10 17:40:36 2014 -0700
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+environment_id="2.0.0@symphony-ssh"
+
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments" \
+	&& -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]; then
+	echo "Using ${environment_id}"
+	. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+
+	if [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]]; then
+		. "${rvm_path:-$HOME/.rvm}/hooks/after_use"
+	fi
+else
+	# If the environment file has not yet been created, use the RVM CLI to select.
+	if ! rvm --create use  "$environment_id"
+		then
+		echo "Failed to create RVM environment '${environment_id}'."
+		exit 1
+	fi
+fi
+
+filename=".rvm.gems"
+if [[ -s "$filename" ]]; then
+	rvm gemset import "$filename"
+fi
+
+
+
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/.simplecov	Sat May 10 17:40:36 2014 -0700
@@ -0,0 +1,9 @@
+# Simplecov config
+
+SimpleCov.start do
+	add_filter 'spec'
+	add_filter 'integration'
+	add_group "Needing tests" do |file|
+		file.covered_percent < 90
+	end
+end
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/README.rdoc	Sat May 10 17:40:36 2014 -0700
@@ -0,0 +1,113 @@
+
+= symphony-ssh
+
+== Description
+
+This is a small collection of base classes used for interacting with
+remote machines over ssh.  With them, you can use AMQP (via Symphony) to
+run batch commands, execute templates as scripts, and perform any
+batch/remoting stuff you can think of without the need of a separate
+client agent.
+
+These classes assume you have a user that can connect and login to
+remote machines using a password-less ssh keypair.  They are not meant
+to be used directly.  Subclass them!
+
+See the rdoc for additional information and examples.
+
+
+== Options
+
+Symphony-ssh uses
+Configurability[https://rubygems.org/gems/configurability] to determine
+behavior.  The configuration is a YAML[http://www.yaml.org/] file. 
+
+	symphony_ssh:
+		path: /usr/bin/ssh
+		user: root
+		key: /path/to/a/private_key.rsa
+		opts:
+		  - -e
+		  - none
+		  - -T
+		  - -x
+		  - -o
+		  - CheckHostIP=no'
+		  - -o
+		  - BatchMode=yes'
+		  - -o
+		  - StrictHostKeyChecking=no
+
+
+
+=== path
+
+The absolute path to the ssh binary.
+
+=== user
+
+The default user to connect to remote hosts with.  This can be
+changes per connection in the AMQP payload.
+
+=== key
+
+An absolute path to a password-less ssh private key.
+
+=== opts
+
+SSH client options, passed to the ssh binary on the command line.  Note
+that the defaults have been tested fairly extensively, these are just
+exposed if you have very specific needs and you know what you're doing.
+
+
+== Installation
+
+    gem install symphony-ssh
+
+
+== Contributing
+
+You can check out the current development source with Mercurial via its
+{project page}[http://bitbucket.org/mahlon/symphony-metronome]. 
+
+After checking out the source, run:
+
+    $ rake
+
+This task will run the tests/specs and generate the API documentation.
+
+If you use {rvm}[http://rvm.io/], entering the project directory will
+install any required development dependencies.
+
+
+== License
+
+Copyright (c) 2014, Mahlon E. Smith and Michael Granger
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the author/s, nor the names of the project's
+  contributors may be used to endorse or promote products derived from this
+  software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Rakefile	Sat May 10 17:40:36 2014 -0700
@@ -0,0 +1,116 @@
+#!/usr/bin/env rake
+# vim: set nosta noet ts=4 sw=4:
+
+require 'rake/clean'
+require 'pathname'
+
+BASEDIR = Pathname( __FILE__ ).dirname.relative_path_from( Pathname.pwd )
+LIBDIR  = BASEDIR + 'lib' + 'symphony'
+CLOBBER.include( 'coverage' )
+
+$LOAD_PATH.unshift( LIBDIR.to_s )
+
+if Rake.application.options.trace
+    $trace = true
+    $stderr.puts '$trace is enabled'
+end
+
+task :default => [ :spec, :docs, :package ]
+
+
+########################################################################
+### P A C K A G I N G
+########################################################################
+
+require 'rubygems'
+require 'rubygems/package_task'
+spec = Gem::Specification.new do |s|
+	s.email        = 'mahlon@martini.nu'
+	s.homepage     = 'http://projects.martini.nu/ruby-modules'
+	s.authors      = [ 'Mahlon E. Smith <mahlon@martini.nu>', 'Michael Granger <ged@faeriemud.org>' ]
+	s.platform     = Gem::Platform::RUBY
+	s.summary      = "Base classes for using Symphony with ssh."
+	s.name         = 'symphony-ssh'
+	s.version      = '0.1.0'
+	s.license      = 'BSD'
+	s.has_rdoc     = true
+	s.require_path = 'lib'
+	s.bindir       = 'bin'
+	s.files        = File.read( __FILE__ ).split( /^__END__/, 2 ).last.split
+	#s.executables  = %w[]
+	s.description  = <<-EOF
+	EOF
+	s.required_rubygems_version = '>= 2.0.3'
+	s.required_ruby_version = '>= 2.0.0'
+
+	s.add_dependency 'symphony', '~> 0.6'
+	s.add_dependency 'net-ssh',  '~> 2.9'
+	s.add_dependency 'net-sftp', '~> 2.1'
+
+	s.add_development_dependency 'rspec',     '~> 3.0'
+	s.add_development_dependency 'simplecov', '~> 0.8'
+end
+
+Gem::PackageTask.new( spec ) do |pkg|
+	pkg.need_zip = true
+	pkg.need_tar = true
+end
+
+
+########################################################################
+### D O C U M E N T A T I O N
+########################################################################
+
+begin
+	require 'rdoc/task'
+
+	desc 'Generate rdoc documentation'
+	RDoc::Task.new do |rdoc|
+		rdoc.name       = :docs
+		rdoc.rdoc_dir   = 'docs'
+		rdoc.main       = "README.rdoc"
+		rdoc.options    = [ '-f', 'fivefish' ]
+		rdoc.rdoc_files = [ 'lib', *FileList['*.rdoc'] ]
+	end
+
+	RDoc::Task.new do |rdoc|
+		rdoc.name       = :doc_coverage
+		rdoc.options    = [ '-C1' ]
+	end
+
+rescue LoadError
+	$stderr.puts "Omitting 'docs' tasks, rdoc doesn't seem to be installed."
+end
+
+
+########################################################################
+### T E S T I N G
+########################################################################
+
+begin
+	require 'rspec/core/rake_task'
+	task :test => :spec
+
+	desc "Run specs"
+	RSpec::Core::RakeTask.new do |t|
+		t.pattern = "spec/**/*_spec.rb"
+	end
+
+	desc "Build a coverage report"
+	task :coverage do
+		ENV[ 'COVERAGE' ] = "yep"
+		Rake::Task[ :spec ].invoke
+	end
+
+rescue LoadError
+	$stderr.puts "Omitting testing tasks, rspec doesn't seem to be installed."
+end
+
+
+########################################################################
+### M A N I F E S T
+########################################################################
+__END__
+lib/symphony/tasks/ssh.rb
+lib/symphony/tasks/sshscript.rb
+README.rdoc
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lib/symphony/tasks/ssh.rb	Sat May 10 17:40:36 2014 -0700
@@ -0,0 +1,166 @@
+#!/usr/bin/env ruby
+# vim: set nosta noet ts=4 sw=4:
+
+require 'shellwords'
+require 'symphony/task' unless defined?( Symphony::Task )
+
+
+### A base class for connecting to remote hosts, running arbitrary
+### commands, and collecting output.
+###
+### This isn't designed to be used directly.  To use this in your
+### environment, you'll want to subclass it, add the behaviors
+### that make sense for you, then super() back to the parent in the
+### #work method.
+###
+### It expects the payload to contain the following keys:
+###
+###    host:    (required) The hostname to connect to
+###    command: (required) The command to run on the remote host
+###    port:    (optional) The port to connect to (defaults to 22)
+###    opts:    (optional) Explicit SSH client options
+###    user:    (optional) The user to connect as (defaults to root)
+###    key:     (optional) The path to an SSH private key
+###
+###
+### Additionally, this class responds to the 'symphony_ssh' configurability
+### key.  Currently, you can set the 'path' argument, which is the
+### full path to the local ssh binary (defaults to '/usr/bin/ssh') and
+### override the default ssh user, key, and client opts.
+###
+### Textual output of the command is stored in the @output instance variable.
+###
+###
+###    require 'symphony'
+###    require 'symphony/tasks/ssh'
+###
+###    class YourTask < Symphony::Task::SSH
+###        timeout 5
+###        subscribe_to 'ssh.command'
+###
+###        def work( payload, metadata )
+###            status = super
+###            puts "Remote host said: %s" % [ @output ]
+###            return status.success?
+###        end
+###    end
+###
+class Symphony::Task::SSH < Symphony::Task
+	extend Configurability
+	config_key :symphony_ssh
+
+	# SSH default options.
+	#
+	CONFIG_DEFAULTS = {
+		:path => '/usr/bin/ssh',
+		:opts => [
+			'-e', 'none',
+			'-T',
+			'-x',
+			'-q',
+			'-o', 'CheckHostIP=no',
+			'-o', 'BatchMode=yes',
+			'-o', 'StrictHostKeyChecking=no'
+		],
+		:user => 'root',
+		:key  => nil
+	}
+
+	# SSH "informative" stdout output that should be cleaned from the
+	# command output.
+	SSH_CLEANUP = %r/Warning: no access to tty|Thus no job control in this shell/
+
+	class << self
+		# The full path to the ssh binary.
+		attr_reader :path
+
+		# A default set of ssh client options when connecting
+		# to remote hosts.
+		attr_reader :opts
+
+		# The default user to use when connecting.  If unset, 'root' is used.
+		attr_reader :user
+
+		# An absolute path to a password-free ssh private key.
+		attr_reader :key
+	end
+
+	### Configurability API.
+	###
+	def self::configure( config=nil )
+		config = self.defaults.merge( config || {} )
+		@path  = config.delete( :path )
+		@opts  = config.delete( :opts )
+		@user  = config.delete( :user )
+		@key   = config.delete( :key )
+		super
+	end
+
+
+	### Perform the ssh connection, passing the command to the pipe
+	### and retreiving any output from the remote end.
+	###
+	def work( payload, metadata )
+		command = payload[ 'command' ]
+		raise ArgumentError, "Missing required option 'command'" unless command
+		raise ArgumentError, "Missing required option 'host'"    unless payload[ 'host' ]
+
+		exitcode = self.open_connection( payload, metadata ) do |reader, writer|
+			self.log.debug "Writing command #{command}..."
+			writer.puts( command )
+			self.log.debug "  closing child's writer."
+			writer.close
+			self.log.debug "  reading from child."
+			reader.read
+		end
+
+		self.log.debug "SSH exited: %d" % [ exitcode ]
+		return exitcode
+	end
+
+
+	#########
+	protected
+	#########
+
+	### Call ssh and yield the remote IO objects to the caller,
+	### cleaning up afterwards.
+	###
+	def open_connection( payload, metadata=nil )
+		raise LocalJumpError, "no block given" unless block_given?
+		@output = ''
+
+		port = payload[ 'port' ] || 22
+		opts = payload[ 'opts' ] || Symphony::Task::SSH.opts
+		user = payload[ 'user' ] || Symphony::Task::SSH.user
+		key  = payload[ 'key'  ] || Symphony::Task::SSH.key
+
+		cmd = []
+		cmd << Symphony::Task::SSH.path
+		cmd += Symphony::Task::SSH.opts
+
+		cmd << '-p' << port.to_s
+		cmd << '-i' << key if key
+		cmd << '-l' << user
+		cmd << payload[ 'host' ]
+		cmd.flatten!
+		self.log.debug "Running SSH command with: %p" % [ Shellwords.shelljoin(cmd) ]
+
+		parent_reader, child_writer = IO.pipe
+		child_reader, parent_writer = IO.pipe
+
+		pid = spawn( *cmd, :out => child_writer, :in => child_reader, :close_others => true )
+		child_writer.close
+		child_reader.close
+
+		self.log.debug "Yielding back to the run block."
+		@output = yield( parent_reader, parent_writer )
+		@output = @output.split("\n").reject{|l| l =~ SSH_CLEANUP }.join
+		self.log.debug "  run block done."
+
+		pid, status = Process.waitpid2( pid )
+		return status
+	end
+
+end # Symphony::Task::SSH
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lib/symphony/tasks/sshscript.rb	Sat May 10 17:40:36 2014 -0700
@@ -0,0 +1,183 @@
+#!/usr/bin/env ruby
+# vim: set nosta noet ts=4 sw=4:
+
+require 'net/ssh'
+require 'net/sftp'
+require 'tmpdir'
+require 'inversion'
+require 'symphony'
+require 'symphony/task'
+
+
+### A base class for connecting to a remote host, then uploading and
+### executing an Inversion templated script.
+###
+### This isn't designed to be used directly.  To use this in your
+### environment, you'll want to subclass it, add the behaviors
+### that make sense for you, then super() back to the parent in the
+### #work method.
+###
+### It expects the payload to contain the following keys:
+###
+###    host:       (required) The hostname to connect to
+###    template:   (required) A path to the Inversion templated script
+###    port:       (optional) The port to connect to (defaults to 22)
+###    user:       (optional) The user to connect as (defaults to root)
+###    key:        (optional) The path to an SSH private key
+###    attributes: (optional) Additional data to attach to the template
+###    nocleanup:  (optional) Leave the remote script after execution? (default to false)
+###
+###
+### Additionally, this class responds to the 'symphony_ssh' configurability
+### key.  Currently, you can override the default ssh user and private key.
+###
+### Textual output of the command is stored in the @output instance variable.
+###
+###
+###    require 'symphony'
+###    require 'symphony/tasks/sshscript'
+###
+###    class YourTask < Symphony::Task::SSHScript
+###        timeout 30
+###        subscribe_to 'ssh.script.*'
+###
+###        def work( payload, metadata )
+###            status = super
+###            puts "Remote script said: %s" % [ @output ]
+###            return status.success?
+###        end
+###    end
+###
+class Symphony::Task::SSHScript < Symphony::Task
+	extend Configurability
+	config_key :symphony_ssh
+
+	# Template config
+	#
+	TEMPLATE_OPTS = {
+		:ignore_unknown_tags => false,
+		:on_render_error     => :propagate,
+		:strip_tag_lines     => true
+	}
+
+	# The defaults to use when connecting via SSH
+	#
+	DEFAULT_SSH_OPTIONS = {
+		:auth_methods            => [ 'publickey' ],
+		:compression             => true,
+		:config                  => false,
+		:keys_only               => true,
+		:paranoid                => false,
+		:global_known_hosts_file => '/dev/null',
+		:user_known_hosts_file   => '/dev/null'
+	}
+
+	# SSH default options.
+	#
+	CONFIG_DEFAULTS = {
+		:user => 'root',
+		:key  => nil
+	}
+
+	class << self
+		# The default user to use when connecting.  If unset, 'root' is used.
+		attr_reader :user
+
+		# An absolute path to a password-free ssh private key.
+		attr_reader :key
+	end
+
+	### Configurability API.
+	###
+	def self::configure( config=nil )
+		config = self.defaults.merge( config || {} )
+		@user = config.delete( :user )
+		@key  = config.delete( :key )
+		super
+	end
+
+
+	### Perform the ssh connection, render the template, send it, and
+	### execute it.
+	###
+	def work( payload, metadata )
+		template   = payload[ 'template' ]
+		attributes = payload[ 'attributes' ] || {}
+		port       = payload[ 'port' ]    || 22
+		user       = payload[ 'user' ]    || Symphony::Task::SSHScript.user
+		key        = payload[ 'key'  ]    || Symphony::Task::SSHScript.key
+		nocleanup  = payload[ 'nocleanup' ]
+
+		raise ArgumentError, "Missing required option 'command'" unless template
+		raise ArgumentError, "Missing required option 'host'"    unless payload[ 'host' ]
+
+		remote_filename = self.make_remote_filename( template )
+		source = self.generate_script( template, attributes )
+
+		ssh_options = DEFAULT_SSH_OPTIONS.merge( :port => port, :keys => [key] )
+		ssh_options.merge!(
+			:logger  => Loggability[ Net::SSH ],
+			:verbose => :debug
+		) if payload[ 'debug' ]
+
+		Net::SSH.start( payload['host'], user, ssh_options ) do |conn|
+			self.log.debug "Uploading script (%d bytes) to %s:%s." %
+				[ source.bytesize, payload['host'], remote_filename ]
+			self.upload_script( conn, source, remote_filename )
+			self.log.debug "  done with the upload."
+
+			self.run_script( conn, remote_filename, nocleanup )
+			self.log.debug "Output was:\n#{@output}"
+		end
+
+		return true
+	end
+
+
+	#########
+	protected
+	#########
+
+	### Generate a unique filename for the script on the remote host,
+	### based on +template+ name.
+	###
+	def make_remote_filename( template )
+		basename = File.basename( template, File.extname(template) )
+		tmpname  = Dir::Tmpname.make_tmpname( basename, rand(10000) )
+
+		return "/tmp/#{tmpname}"
+	end
+
+
+	### Generate a script by loading the script +template+, populating it with
+	### +attributes+, and returning the rendered output.
+	###
+	def generate_script( template, attributes )
+		tmpl = Inversion::Template.load( template, TEMPLATE_OPTS )
+		tmpl.attributes.merge!( attributes )
+		tmpl.task = self
+
+		return tmpl.render
+	end
+
+
+	### Upload the templated +source+ via the ssh +conn+ to an
+	### executable file named +remote_filename+.
+	###
+	def upload_script( conn, source, remote_filename )
+		conn.sftp.file.open( remote_filename, "w", 0755 ) do |fh|
+			fh.print( source )
+		end
+	end
+
+
+	### Run the +remote_filename+ via the ssh +conn+.  The script
+	### will be deleted automatically unless +nocleanup+ is true.
+	###
+	def run_script( conn, remote_filename, nocleanup=false )
+		@output = conn.exec!( remote_filename )
+		conn.exec!( "rm #{remote_filename}" ) unless nocleanup
+	end
+
+end # Symphony::Task::SSHScript
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/spec/helpers.rb	Sat May 10 17:40:36 2014 -0700
@@ -0,0 +1,43 @@
+#!/usr/bin/ruby
+# coding: utf-8
+# vim: set nosta noet ts=4 sw=4:
+
+require 'pathname'
+
+BASEDIR = Pathname( __FILE__ ).dirname.parent
+LIBDIR  = BASEDIR + 'lib'
+
+$LOAD_PATH.unshift( LIBDIR.to_s )
+
+# SimpleCov test coverage reporting; enable this using the :coverage rake task
+require 'simplecov' if ENV['COVERAGE']
+
+require 'loggability'
+require 'loggability/spechelpers'
+require 'configurability'
+require 'configurability/behavior'
+require 'rspec'
+
+require 'symphony'
+
+Loggability.format_with( :color ) if $stdout.tty?
+
+
+### RSpec helper functions.
+module Loggability::SpecHelpers
+end
+
+
+### Mock with RSpec
+RSpec.configure do |config|
+	config.run_all_when_everything_filtered = true
+	config.filter_run :focus
+	# config.order = 'random'
+	config.expect_with( :rspec )
+	config.mock_with( :rspec ) do |mock|
+		mock.syntax = :expect
+	end
+
+	config.include( Loggability::SpecHelpers )
+end
+