chunker/lib/chunker.rb
author mahlon
Sun, 09 Nov 2008 00:27:36 +0000
branchruby-modules
changeset 2 e5c705047540
parent 1 9e127bf6e84f
child 4 01a3332bfe0a
permissions -rw-r--r--
* Rename 'markers' to 'token' * Fix up the Rakefile's gem generation * Add LICENSE * Add a real README
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
2
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
     1
#!/usr/bin/ruby
1
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
     2
#
2
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
     3
# Chunker: A convenience library for parsing __END__ tokens consistently.
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
     4
#
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
     5
# == Version
1
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
     6
#
2
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
     7
#	$Id$
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
     8
#
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
     9
# == Author
1
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    10
#
2
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
    11
# * Mahlon E. Smith <mahlon@martini.nu>
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
    12
#
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
    13
# :include: LICENSE
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
    14
#
1
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    15
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    16
### Namespace for the datablock parser.
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    17
###
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    18
module Chunker
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    19
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    20
	require 'strscan'
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    21
	require 'stringio'
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    22
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    23
	# SVN Revision
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    24
	#
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    25
	SVNRev = %q$Rev$
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    26
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    27
	# SVN Id
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    28
	#
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    29
	SVNId = %q$Id$
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    30
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    31
	# Package version
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    32
	#
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    33
	VERSION = '0.1'
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    34
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    35
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    36
	### Parser class for __END__ data blocks.
2
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
    37
	### Find each __TOKEN__ within the __END__, and put each into a
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
    38
	### DATA_TOKEN constant within the namespace that included us.
1
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    39
	###
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    40
	class DataParser
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    41
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    42
		# The mark for a DATA block.
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    43
		#
2
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
    44
		END_TOKEN = /^__END__\r?\n/
1
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    45
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    46
		# The mark for a 'sub' block.
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    47
		#
2
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
    48
		CHUNK_TOKEN = /^__([A-Z\_0-9]+)__\r?\n/
1
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    49
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    50
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    51
		### Constructor: Given a +klass+ and an +io+ to the class file,
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    52
		### extract the data blocks and install constants.
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    53
		###
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    54
		def initialize( klass, io )
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    55
			io.open if io.closed?
2
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
    56
			end_string = io.read.split( END_TOKEN, 2 ).last
1
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    57
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    58
			@klass   = klass
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    59
			@scanner = StringScanner.new( end_string )
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    60
			io.close
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    61
2
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
    62
			if @scanner.check_until( CHUNK_TOKEN )
1
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    63
				# put each chunk into its own constant
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    64
				self.extract_blocks
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    65
			else
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    66
				# no sub blocks, put the whole mess into DATA_END
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    67
				@klass.const_set( :DATA_END, StringIO.new( end_string ) )
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    68
			end
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    69
		end
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    70
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    71
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    72
		#########
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    73
		protected
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    74
		#########
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    75
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    76
		### Parse the current +io+ for data blocks, set contents to
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    77
		### IO constants in the including class.
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    78
		###
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    79
		def extract_blocks
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    80
			label = nil
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    81
2
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
    82
			while @scanner.scan_until( CHUNK_TOKEN ) and ! @scanner.eos?
1
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    83
				data = ''
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    84
2
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
    85
				# First pass, __END__ contents (until next token, instead
1
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    86
				# of entire data block.)
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    87
				#
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    88
				if label.nil?
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    89
					label = 'END'
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    90
					data  = @scanner.pre_match
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    91
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    92
					@scanner.pos = self.next_position
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    93
				else
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    94
					label = @scanner[1]
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    95
2
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
    96
					if data = @scanner.scan_until( CHUNK_TOKEN )
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
    97
						# Pull the next token text out of the data, set up the next pass
1
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    98
						#
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
    99
						data         = data[ 0, data.length - @scanner[0].length ]
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   100
						@scanner.pos = self.next_position
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   101
					else
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   102
						# No additional blocks
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   103
						#
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   104
						data = @scanner.rest
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   105
					end
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   106
				end
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   107
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   108
				# Add the IO constant to the class that included me.
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   109
				#
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   110
				@klass.const_set( "DATA_#{label}".to_sym, StringIO.new( data ) )
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   111
			end
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   112
		end
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   113
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   114
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   115
		### Return the next scanner position for searching.
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   116
		###
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   117
		def next_position
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   118
			return @scanner.pos - @scanner[0].length
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   119
		end
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   120
	end
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   121
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   122
2
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
   123
	### Hook included: Find the file path for how we arrived here, and open
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
   124
	### it as an IO object.  Parse the IO for data block tokens.
1
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   125
	###
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   126
    def self.included( klass )
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   127
		# klass.instance_eval{ __FILE__ }   awww, nope.
2
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
   128
		# __FILE__ won't work here, so we find the filename via caller().
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
   129
		#
1
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   130
		io = File.open( caller(1).last.sub(/:.*?$/, ''), 'r' )
2
e5c705047540 * Rename 'markers' to 'token'
mahlon
parents: 1
diff changeset
   131
1
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   132
		DataParser.new( klass, io )
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   133
    end
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   134
end
9e127bf6e84f Initial commit of chunker, a ruby module to aid with data blocks.
mahlon
parents:
diff changeset
   135