From 2952e2cbd7c5b9a1006f817a49ad845e32e1c8fb Mon Sep 17 00:00:00 2001 From: mahlon Date: Wed, 21 Jun 2023 03:02:52 +0000 Subject: [PATCH] Got all the header parsing and matching working. Altered core logic to just be: - rules that run before a global filters - N global filter(s) - rules that run after filters All steps are optional. FossilOrigin-Name: 250b5d22e0387865e13a4adee9dadd122583311bde1d015eefb3321c525827be --- config.yml | 31 +++++---- src/lib/config.nim | 16 ++--- src/lib/message.nim | 159 ++++++++++++++++++++++++++++++++++---------- src/sieb.nim | 28 +++++--- 4 files changed, 168 insertions(+), 66 deletions(-) diff --git a/config.yml b/config.yml index 8a69bcb..d02f020 100644 --- a/config.yml +++ b/config.yml @@ -5,14 +5,19 @@ # Default, no logging. Relative to homedir. #logfile: sieb.log -## Filter message before rules -pre_filter: - - 'reformail -A "X-Sieb: Pre-filtered!"' -# - bogofilter +## Rules tried before global filtering +early_rules: + - + headers: + to: mahlon@(laika|ravn) + received: .*sendgrid.* + filter: + - [ reformail, -A, "X-Sieb: This matched." ] + deliver: .whatever -## Filter message after rules -#post_filter: -# - bogofilter +## Filter message before additional rules. +filter: + - [ reformail, -A, "X-Sieb: That shit totally matched" ] ## Ordered, top down, first match wins. ## Headers are lowercased. Multiple matches are AND'ed. @@ -20,12 +25,12 @@ pre_filter: ## Delivery default is ~/Maildir, any set value is an auto-created maildir under ## that path. ## -rules: - - - headers: - x-what: pcre-matcher - poonie: pcre-matcher - deliver: .whatever +# rules: +# - +# headers: +# x-what: pcre-matcher +# poonie: pcre-matcher +# deliver: .whatever # # Magic "TO" which means To: OR Cc: # - diff --git a/src/lib/config.nim b/src/lib/config.nim index 905b353..db8ad0d 100644 --- a/src/lib/config.nim +++ b/src/lib/config.nim @@ -34,17 +34,17 @@ const CONFFILES = @[ ############################################################# type - rule = object - headers {.defaultVal: initTable[string, string]()}: Table[ string, string ] - deliver {.defaultVal: "Maildir"}: string - filter {.defaultVal: ""}: string + Rule* = object + headers* {.defaultVal: initTable[string, string]()}: Table[ string, string ] + deliver* {.defaultVal: ""}: string + filter* {.defaultVal: @[]}: seq[ seq[string] ] # Typed configuration file layout for YAML loading. Config* = object - logfile* {.defaultVal: "".}: string - pre_filter* {.defaultVal: @[]}: seq[string] - post_filter* {.defaultVal: @[]}: seq[string] - rules* {.defaultVal: @[]}: seq[rule] + logfile* {.defaultVal: "".}: string + filter* {.defaultVal: @[]}: seq[ seq[string] ] + early_rules* {.defaultVal: @[]}: seq[Rule] + rules* {.defaultVal: @[]}: seq[Rule] ############################################################# diff --git a/src/lib/message.nim b/src/lib/message.nim index ab90b55..63322f7 100644 --- a/src/lib/message.nim +++ b/src/lib/message.nim @@ -12,11 +12,15 @@ import std/os, std/osproc, std/posix, + std/re, std/streams, std/strformat, + std/strutils, + std/tables, std/times import + config, util @@ -27,8 +31,8 @@ import const OWNERDIRPERMS = { fpUserExec, fpUserWrite, fpUserRead } OWNERFILEPERMS = { fpUserWrite, fpUserRead } - # FILTERPROCOPTS = { poUsePath } - FILTERPROCOPTS = { poUsePath, poEvalCommand } + FILTERPROCOPTS = { poUsePath } + # FILTERPROCOPTS = { poUsePath, poEvalCommand } BUFSIZE = 8192 # reading and writing buffer size @@ -49,7 +53,7 @@ type Maildir* = ref object type Message* = ref object basename: string dir: Maildir - headers: seq[ tuple[ header: string, value: seq[string] ] ] + headers*: Table[ string, seq[string] ] path: string stream: FileStream @@ -107,7 +111,6 @@ proc newMessage*( dir: Maildir ): Message = result.dir = dir result.basename = $now.toUnixFloat & '.' & $getCurrentProcessID() & '.' & $msgcount & '.' & $hostname result.path = joinPath( result.dir.tmp, result.basename ) - result.headers = @[] try: debug "Opening new message at {result.path}".fmt @@ -118,7 +121,7 @@ proc newMessage*( dir: Maildir ): Message = proc open*( msg: Message ) = - ## Open (or re-open) a Message file stream. + ## Open (or re-open) a Message file stream for reading. msg.stream = msg.path.openFileStream @@ -141,16 +144,15 @@ proc delete*( msg: Message ) = msg.path = "" -proc writeStdin*( msg: Message ) = - ## Streams stdin to the message file, returning how - ## many bytes were written. +proc writeStdin*( msg: Message ): Message = + ## Streams stdin to the message file, returning the Message + ## object for chaining. let input = stdin.newFileStream - var buf = input.readStr( BUFSIZE ) - var total = buf.len - msg.stream.write( buf ) + var total = 0 + result = msg - while buf != "" and buf.len == BUFSIZE: - buf = input.readStr( BUFSIZE ) + while not input.atEnd: + let buf = input.readStr( BUFSIZE ) total = total + buf.len msg.stream.write( buf ) msg.stream.flush @@ -158,40 +160,36 @@ proc writeStdin*( msg: Message ) = debug "Wrote {total} bytes from stdin".fmt -proc filter*( orig_msg: Message, cmd: string ): Message = +proc filter*( orig_msg: Message, cmd: seq[string] ): Message = ## Filter message content through an external program, ## returning a new Message if successful. + result = orig_msg try: var buf: string - # let command = cmd.split - # let process = command[0].startProcess( - # args = command[1..(command.len-1)], - # options = FILTERPROCOPTS - # ) + let process = cmd[0].startProcess( + args = cmd[1..(cmd.len-1)], + options = FILTERPROCOPTS + ) - let process = cmd.startProcess( options = FILTERPROCOPTS ) + debug "Running filter: {cmd}".fmt + # let process = cmd.startProcess( options = FILTERPROCOPTS ) # Read from the original message, write to the filter # process in chunks. # orig_msg.open - buf = orig_msg.stream.readStr( BUFSIZE ) - process.inputStream.write( buf ) - process.inputStream.flush - while buf != "" and buf.len == BUFSIZE: + while not orig_msg.stream.atEnd: buf = orig_msg.stream.readStr( BUFSIZE ) process.inputStream.write( buf ) process.inputStream.flush # Read from the filter process until EOF, send to the # new message in chunks. + # process.inputStream.close let new_msg = newMessage( orig_msg.dir ) - buf = process.outputStream.readStr( BUFSIZE ) - new_msg.stream.write( buf ) - new_msg.stream.flush - while buf != "" and buf.len == BUFSIZE: + while not process.outputStream.atEnd: buf = process.outputStream.readStr( BUFSIZE ) new_msg.stream.write( buf ) new_msg.stream.flush @@ -199,19 +197,108 @@ proc filter*( orig_msg: Message, cmd: string ): Message = let exitcode = process.waitForExit debug "Filter exited: {exitcode}".fmt process.close - orig_msg.delete - result = new_msg + if exitcode == 0: + new_msg.stream.close + orig_msg.delete + result = new_msg + else: + debug "Unable to filter message: non-zero exit code".fmt except OSError as err: debug "Unable to filter message: {err.msg}".fmt - result = orig_msg -# FIXME: header parsing to tuples -# - open file -# - skip lines that don't match headers -# - unwrap multiline headers -# - store header, add value to seq of strings +proc parseHeaders*( msg: Message ) = + ## Walk the RFC2822 headers, placing them into memory. + ## This 'unwraps' multiline headers, and allows for duplicate headers. + debug "Parsing message headers." + msg.headers = initTable[ string, seq[string] ]() + msg.open + + var + line = "" + header = "" + value = "" + + while msg.stream.readLine( line ): + if line == "": # Stop when headers are done. + if header != "": + if msg.headers.hasKey( header ): + msg.headers[ header ].add( value ) + else: + msg.headers[ header ] = @[ value ] + break + + # Fold continuation line + # + if line.startsWith( ' ' ) or line.startsWith( '\t' ): + line = line.replace( re"^\s+" ) + value = value & ' ' & line + + # Header start + # + else: + var matches: array[ 2, string ] + if line.match( re"^([\w\-]+):\s*(.*)", matches ): + if header != "": + if msg.headers.hasKey( header ): + msg.headers[ header ].add( value ) + else: + msg.headers[ header ] = @[ value ] + ( header, value ) = ( matches[0].toLower, matches[1] ) +# FIXME: magic TO +proc walkRules*( msg: var Message, rules: seq[Rule], default: Maildir ): bool = + ## Evaluate each rule against the Message, returning true + ## if there was a valid match found. + msg.parseHeaders + result = false + + for rule in rules: + var match = false + + block thisRule: + for header, regexp in rule.headers: + let header_chk = header.toLower + var hmatch = false + + debug " checking header \"{header}\"".fmt + if msg.headers.hasKey( header_chk ): + for val in msg.headers[ header_chk ]: + try: + hmatch = val.match( regexp.re ) + if hmatch: + debug " match on \"{regexp}\"".fmt + break # a single multi-header is sufficient + except RegexError as err: + debug " invalid regexp \"{regexp}\" ({err.msg}), skipping".fmt.replace( "\n", " " ) + break thisRule + + # Did any of the (possibly) multi-header values match? + if hmatch: + match = true + else: + debug " no match, skipping others" + break thisRule + + else: + debug " nonexistent header, skipping others" + break thisRule + + result = match + + + if result: + debug "Rule match!" + for filter in rule.filter: msg = msg.filter( filter ) + + var deliver: Maildir + if rule.deliver != "": + deliver = default.subDir( rule.deliver ) + else: + deliver = default + + msg.save( deliver ) + diff --git a/src/sieb.nim b/src/sieb.nim index 9b69cad..d487237 100644 --- a/src/sieb.nim +++ b/src/sieb.nim @@ -1,8 +1,7 @@ # vim: set et nosta sw=4 ts=4 : import - std/os, - std/strformat + std/os import lib/config, @@ -20,11 +19,22 @@ let conf = get_config( opts.config ) default = newMaildir( joinPath( home, "Maildir" ) ) -# let dest = default.subDir( "woo" ) -var msg = default.newMessage -msg.writeStdin -for filter in conf.pre_filter: - debug "Running pre-filter: {filter}".fmt - msg = msg.filter( filter ) -msg.save() + +# Create a new message under Maildir/tmp, and stream stdin to it. +var msg = default.newMessage.writeStdin + +# If there are "early rules", parse the message now and walk those. +if conf.early_rules.len > 0: + if msg.walkRules( conf.early_rules, default ): quit( 0 ) + +# Apply any configured global filtering. +for filter in conf.filter: msg = msg.filter( filter ) + +# Walk the rules, and if nothing hits, deliver to fallthrough. +if conf.rules.len > 0: + if not msg.walkRules( conf.rules, default ): msg.save +else: + msg.save + +quit( 0 )