From a938cf045aa36bdc55be209ea79797ca7f9db481 Mon Sep 17 00:00:00 2001 From: mahlon Date: Thu, 29 Jun 2023 12:58:42 +0000 Subject: [PATCH] Multiple changes. - Use the ARC memory model for release builds. - Move logfile to a command line switch, to avoid chicken-and-egg logging when failing to parse YAML. - Fix logic bug: Stop processing rules on a good match - Add performance timer and memory used when logging to file - Rename 'headers' to 'match' in configuration file, more intention revealing - Add logger explicit flock (unlock) at process exit FossilOrigin-Name: 7c439d99044b8d725c2dc1a806eec14fff7f0675afb14920de3c7c2581907640 --- Makefile | 4 ++-- config.yml | 25 +++++++++++++++--------- src/lib/config.nim | 12 +++++------- src/lib/logging.nim | 18 ++++++++++++------ src/lib/message.nim | 24 +++++++++++++---------- src/lib/util.nim | 46 ++++++++++++++++++++++++++++++++++----------- src/sieb.nim | 40 +++++++++++++++++++++++++-------------- 7 files changed, 110 insertions(+), 59 deletions(-) diff --git a/Makefile b/Makefile index b9272e2..570d207 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ dependencies: development: ${FILES} # can use gdb with this... - nim --verbosity:2 --debugInfo --assertions:on --linedir:on -d:debug -d:nimTypeNames --nimcache:.cache c ${FILES} + nim --verbosity:2 --debugInfo --assertions:on --stacktrace:on --linedir:on -d:debug -d:nimTypeNames --nimcache:.cache c ${FILES} mv src/sieb . debugger: ${FILES} @@ -19,7 +19,7 @@ debugger: ${FILES} mv src/sieb . release:dependencies ${FILES} - nim -d:release -d:strip --passc:-flto --opt:speed --nimcache:.cache c ${FILES} + nim -d:release -d:strip --mm:arc -d:lto --opt:speed --nimcache:.cache c ${FILES} mv src/sieb . docs: diff --git a/config.yml b/config.yml index 9dc57d5..3aecf19 100644 --- a/config.yml +++ b/config.yml @@ -2,15 +2,13 @@ # Example Sieb configuration file. # -# Default, no logging. Relative to homedir. -logfile: sieb.log -## Rules tried before global filtering +# Rules tried before global filtering. +# early_rules: - - headers: - TO: mhlon@(laika|ravn) - received: .*sendgrid.* + match: + TO: ahlon@(laika|ravn|martini) filter: - [ reformail, -A, "X-Sieb: This matched." ] deliver: .whatever @@ -27,9 +25,18 @@ filter: ## rules: - - headers: - x-sieb: global - deliver: .whoas + match: + x-one: global + - + match: + Subject: .*\s+corepacket\s+.* + deliver: .balls + filter: + - [ reformail, -A, "X-Sieb: Boom!" ] + - + match: + x-three: global + # # Magic "TO" which means To: OR Cc: # - diff --git a/src/lib/config.nim b/src/lib/config.nim index db8ad0d..f296028 100644 --- a/src/lib/config.nim +++ b/src/lib/config.nim @@ -11,7 +11,6 @@ import std/os, std/streams, - std/strformat, std/tables, yaml/parser, yaml/serialization @@ -35,13 +34,12 @@ const CONFFILES = @[ type Rule* = object - headers* {.defaultVal: initTable[string, string]()}: Table[ string, string ] + match* {.defaultVal: initTable[string, string]()}: Table[ string, string ] deliver* {.defaultVal: ""}: string filter* {.defaultVal: @[]}: seq[ seq[string] ] # Typed configuration file layout for YAML loading. Config* = object - logfile* {.defaultVal: "".}: string filter* {.defaultVal: @[]}: seq[ seq[string] ] early_rules* {.defaultVal: @[]}: seq[Rule] rules* {.defaultVal: @[]}: seq[Rule] @@ -53,7 +51,7 @@ type proc parse( path: string ): Config = ## Return a parsed configuration from yaml. - debug "Using configuration at: {path}".fmt + "Using configuration at: $#".debug( path ) let stream = newFileStream( path ) try: stream.load( result ) @@ -61,18 +59,18 @@ proc parse( path: string ): Config = debug err.msg return Config() # return empty default, it could be "half parsed" except YamlConstructionError as err: - debug err.msg + err.msg.debug return Config() finally: stream.close -proc get_config*( path: string ): Config = +proc getConfig*( path: string ): Config = ## Choose a configuration file for parsing, or if there are ## none available, return an empty config. if path != "": if not path.fileExists: - debug "Configfile \"{path}\" unreadable, ignoring.".fmt + "Configfile \"$#\" unreadable, ignoring.".debug( path ) return return parse( path ) diff --git a/src/lib/logging.nim b/src/lib/logging.nim index 16b1564..c56719f 100644 --- a/src/lib/logging.nim +++ b/src/lib/logging.nim @@ -8,6 +8,8 @@ ############################################################# import + std/math, + std/monotimes, std/os, std/posix, std/times @@ -19,6 +21,7 @@ import type Logger = object fh: File + start: MonoTime ############################################################# @@ -32,21 +35,24 @@ var logger*: Logger # M E T H O D S ############################################################# -proc createLogger*( path: string ): void = +proc createLogger*( parentdir: string, filename: string ): void = ## Get in line to open a write lock to the configured logfile at +path+. ## This will block until it can get an exclusive lock. - let path = joinPath( getHomeDir(), path ) - logger = Logger() - logger.fh = path.open( fmAppend ) + let path = joinPath( parentdir, filename ) + logger = Logger() + logger.fh = path.open( fmAppend ) + logger.start = getMonoTime() # Wait for exclusive lock. discard logger.fh.getFileHandle.lockf( F_LOCK, 0 ) - logger.fh.writeLine "\n-------------------------------------------------------------------" - logger.fh.writeLine now().utc + logger.fh.writeLine "\n", now().utc, " ------------------------------------------------------------------" proc close*( l: Logger ): void = ## Release the lock and close/flush the file. + let duration = float( (getMonoTime() - l.start).inNanoSeconds ) / 1_000_000 # ms + let memUsed = ( getOccupiedMem() / 1024 ).round( 2 ) + l.fh.writeLine "Completed in ", duration.round( 2 ), "ms, using ", memUsed, "Kb" discard l.fh.getFileHandle.lockf( F_ULOCK, 0 ) l.fh.close() diff --git a/src/lib/message.nim b/src/lib/message.nim index 988af93..a19bd5e 100644 --- a/src/lib/message.nim +++ b/src/lib/message.nim @@ -58,8 +58,9 @@ type Message* = ref object stream: FileStream -# Count messages generated during a run. -var msgcount = 0 +var + msgcount = 0 # Count messages generated during a run. + msgId = "" # The parsed Message-ID ############################################################# @@ -212,7 +213,6 @@ proc filter*( orig_msg: Message, cmd: seq[string] ): Message = proc parseHeaders*( msg: Message ) = ## Walk the RFC2822 headers, placing them into memory. ## This 'unwraps' multiline headers, and allows for duplicate headers. - let preparsed = not msg.headers.isNil msg.headers = initTable[ string, seq[string] ]() msg.open @@ -232,8 +232,9 @@ proc parseHeaders*( msg: Message ) = # Fold continuation line # - if line.startsWith( ' ' ) or line.startsWith( '\t' ): - line = line.replace( re"^\s+" ) + let wsp = re"^\s+" + if line.match( wsp ): + line = line.replace( wsp ) value = value & ' ' & line # Header start @@ -248,9 +249,11 @@ proc parseHeaders*( msg: Message ) = msg.headers[ header ] = @[ value ] ( header, value ) = ( matches[0].toLower, matches[1] ) - "Parsed message headers.".debug - if msg.headers.hasKey( "message-id" ): - "Message-ID is \"$#\"".debug( msg.headers[ "message-id" ] ) + if msgId == "": + "Parsed message headers.".debug + if msg.headers.hasKey( "message-id" ): + msgId = msg.headers[ "message-id" ][0] + "Message-ID is \"$#\"".debug( msg.headers[ "message-id" ] ) proc evalRules*( msg: var Message, rules: seq[Rule], default: Maildir ): bool = @@ -262,9 +265,9 @@ proc evalRules*( msg: var Message, rules: seq[Rule], default: Maildir ): bool = for rule in rules: var match = false - if rule.headers.len > 0: "Evaluating rule...".debug + if rule.match.len > 0: "Evaluating rule...".debug block thisRule: - for header, regexp in rule.headers: + for header, regexp in rule.match: let header_chk = header.toLower var hmatch = false @@ -320,4 +323,5 @@ proc evalRules*( msg: var Message, rules: seq[Rule], default: Maildir ): bool = deliver = default msg.save( deliver ) + return # stop processing additional rules diff --git a/src/lib/util.nim b/src/lib/util.nim index 6769da8..522d060 100644 --- a/src/lib/util.nim +++ b/src/lib/util.nim @@ -35,20 +35,37 @@ const -d --debug: Debug: Be verbose while parsing. + -g --generate: + Emit an example configuration file to stdout. + -h --help: Help. You're lookin' at it. + -l --log: + A file to record actions to, relative to $HOME/Maildir. + -v --version: Display version number. """ + EXAMPLECONFIG = """ +sdfdsfsdfsdfsdfdsfdf FIXME: FIXME WJSDFJKSDFKSDF + """ ############################################################# # T Y P E S ############################################################# type Opts = object - config*: string # The path to an explicit configuration file. - debug*: bool # Explain what's being done. + config*: string # The path to an explicit configuration file. + debug*: bool # Explain what's being done. + logfile*: string # Log actions to disk. + + +############################################################# +# G L O B A L E X P O R T S +############################################################# + +var opts*: Opts ############################################################# @@ -74,24 +91,25 @@ proc deferral*( msg: string ) = proc debug*( msg: string, args: varargs[string, `$`] ) = ## Emit +msg+ if debug mode is enabled, coercing arguments into a string for ## formatting. - if defined( debug ) or not logger.closed: + if opts.debug or not logger.closed: var str = msg % args - if defined( debug ): echo str + if opts.debug: echo str if not logger.closed: str.log -proc parse_cmdline*: Opts = +proc parseCmdline*() = ## Populate the opts object with the user's preferences. # Config object defaults. # - result = Opts( + opts = Opts( config: "", - debug: false + debug: false, + logfile: "" ) # always set debug mode if development build. - result.debug = defined( debug ) + opts.debug = defined( debug ) for kind, key, val in getopt(): case kind @@ -102,15 +120,22 @@ proc parse_cmdline*: Opts = of cmdLongOption, cmdShortOption: case key of "conf", "c": - result.config = val + opts.config = val of "debug", "d": - result.debug = true + opts.debug = true + + of "generate", "g": + echo EXAMPLECONFIG + quit( 0 ) of "help", "h": echo USAGE quit( 0 ) + of "log", "l": + opts.logfile = val + of "version", "v": echo "Sieb " & VERSION quit( 0 ) @@ -119,4 +144,3 @@ proc parse_cmdline*: Opts = of cmdEnd: assert( false ) # shouldn't reach here - diff --git a/src/sieb.nim b/src/sieb.nim index d1bdb1d..a28d487 100644 --- a/src/sieb.nim +++ b/src/sieb.nim @@ -1,6 +1,11 @@ # vim: set et nosta sw=4 ts=4 : +############################################################# +# I M P O R T S +############################################################# + import + std/exitprocs, std/os import @@ -9,37 +14,43 @@ import lib/message, lib/util -# /home/mahlon/repo/sieb/src/sieb.nim(30) sieb -# /home/mahlon/.choosenim/toolchains/nim-1.6.10/lib/system/io.nim(759) open -# Error: unhandled exception: cannot open: /home/mahlon/ [IOError] - -# TODO: timer/performance -# TODO: more performant debug -# TODO: generate default config? +############################################################# +# S E T U P +############################################################# # Without this, we got nuthin'! if not existsEnv( "HOME" ): - deferral "Unable to determine HOME from environment." + deferral "Fatal: Unable to determine HOME from environment." + +# Populate $opts +parseCmdline() let home = getHomeDir() - opts = parse_cmdline() - conf = get_config( opts.config ) default = newMaildir( joinPath( home, "Maildir" ) ) -if conf.logfile != "": - createLogger( conf.logfile ) +# Open the optional log file. +if opts.logfile != "": createLogger( default.path, opts.logfile ) + +# Exit hook - clean up any open logger filehandle. +var finalTasks = proc: void = + if not logger.closed: logger.close +finalTasks.addExitProc -# FIXME: at exit? -# ... if logger not nil logger close +############################################################# +# M A I N +############################################################# +# Parse the YAML ruleset. +let conf = getConfig( opts.config ) # Create a new message under Maildir/tmp, and stream stdin to it. var msg = default.newMessage.writeStdin # If there are "early rules", parse the message now and walk those. +# if conf.early_rules.len > 0: if msg.evalRules( conf.early_rules, default ): quit( 0 ) @@ -47,6 +58,7 @@ if conf.early_rules.len > 0: for filter in conf.filter: msg = msg.filter( filter ) # Walk the rules, and if nothing hits, deliver to fallthrough. +# if conf.rules.len > 0: if not msg.evalRules( conf.rules, default ): msg.save else: