# HG changeset patch # User Mahlon E. Smith # Date 1519093345 28800 # Node ID 475c9942eb153f07c48d0016b9a1be136066c425 # Parent 252cdb26f76b1bb0ec161fb305d61ca546b5978f Multiple changes: - Make the 'release' build the default. - Add a configurable socket timeout parameter. - Make the table name configurable. - Add usage docs to the README. diff -r 252cdb26f76b -r 475c9942eb15 Makefile --- a/Makefile Sun Feb 18 23:40:34 2018 -0800 +++ b/Makefile Mon Feb 19 18:22:25 2018 -0800 @@ -1,7 +1,7 @@ FILES = netdata_tsrelay.nim -default: development +default: release debug: ${FILES} nim --assertions:on --nimcache:.cache c ${FILES} diff -r 252cdb26f76b -r 475c9942eb15 README.md --- a/README.md Sun Feb 18 23:40:34 2018 -0800 +++ b/README.md Mon Feb 19 18:22:25 2018 -0800 @@ -14,18 +14,17 @@ Installation ------------ -You'll need a working [Nim](http://nim-lang.org) build environment to -create the binary. +You'll need a working [Nim](http://nim-lang.org) build environment and +PostgreSQL development headers to compile the binary. -Simply run `make release` to produce the binary. Put it wherever you -please. +Simply run `make` to build it. Put it wherever you please. Configuration ------------- There are a few assumptions that should be satisfied before running -this. +this successfully. ### Database setup @@ -39,25 +38,28 @@ ); ``` -Index it however you please based on how you intend to query the data, -including JSON functional indexing, etc. See PostgreSQL documentation -for details. +Index it based on how you intend to query the data, including JSON +functional indexing, etc. See PostgreSQL documentation for details. Strongly encouraged: Promote this table to a Timescale "hypertable". -See Timescale docs for that, but a quick example to partition -automatically at weekly boundaries would look something like: +See [Timescale](http://timescale.com) docs for that, but a quick example +to partition automatically at weekly boundaries would look something +like: ```sql SELECT create_hypertable( 'netdata', 'time', chunk_time_interval => 604800000000 ); ``` +Timescale also has some great examples and advice for efficient [JSON +indexing](http://docs.timescale.com/v0.8/using-timescaledb/schema-management#json) +and queries. ### Netdata You'll likely want to pare down what netdata is sending. Here's an -example configuration for `netdata.conf` -- you will want to season -this to taste. +example configuration for `netdata.conf` -- season this to taste (what +charts to send and frequency.) ``` [backend] @@ -67,8 +69,66 @@ data source = average destination = machine-where-netdata-tsrelay-lives:14866 prefix = n - update every = 10 - buffer on failures = 6 + update every = 60 + buffer on failures = 5 send charts matching = !cpu.cpu* !ipv6* !users* nfs.rpc net.* net_drops.* net_packets.* !system.interrupts* system.* disk.* disk_space.* disk_ops.* mem.* ``` + +Running the Relay +----------------- + +### Options + + * [-q|--quiet]: Quiet mode. No output at all. Ignored if -d is supplied. + * [-d|--debug]: Debug mode. Show incoming data. + * [--dbopts]: PostgreSQL connection information. (See below for more details.) + * [-h|--help]: Display quick help text. + * [--listen-addr]: A specific IP address to listen on. Defaults to INADDR_ANY. + * [--listen-port]: The port to listen for netdata JSON streams. + Default is 14866. + * [-T|--dbtable]: Change the table name to insert to. Defaults to **netdata**. + * [-t|--timeout]: Maximum time in milliseconds to wait for data. Slow + connections may need to increase this from the default 500 ms. + * [-v|--version]: Show version. + + +**Notes** + +Nim option parsing might be slightly different than what you're used to. +Flags that require arguments must include an '=' or ':' character. + + * --timeout=1000 *valid* + * --timeout:1000 *valid* + * --t:1000 *valid* + * --timeout 1000 *invalid* + * -t 1000 *invalid* + +All database connection options are passed as a key/val string to the +*dbopts* flag. The default is: + + "host=localhost port=5432 dbname=netdata user=netdata application_name=netdata-tsrelay" + +Reference +https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ- +PARAMKEYWORDS for all available options (including how to store +passwords in a seperate file, enable SSL mode, etc.) + + +### Daemonizing + +Use a tool of your choice to run this at system +startup in the background. My personal preference is +[daemontools](https://cr.yp.to/daemontools.html), but I won't judge you +if you use something else. + +Here's an example using the simple +[daemon](https://www.freebsd.org/cgi/man.cgi?query=daemon&apropos=0&sektion=8&manpath=FreeBSD+11.0-RELEASE+and+Ports&arch=default&format=html) wrapper tool: + + # daemon \ + -o /var/log/netdata_tsrelay.log \ + -p /var/run/netdata_tsrelay.pid \ + -u nobody -cr \ + /usr/local/bin/netdata_tsrelay \ + --dbopts="dbname=metrics user=metrics host=db-master port=6432 application_name=netdata-tsrelay" + diff -r 252cdb26f76b -r 475c9942eb15 netdata_tsrelay.nim --- a/netdata_tsrelay.nim Sun Feb 18 23:40:34 2018 -0800 +++ b/netdata_tsrelay.nim Mon Feb 19 18:22:25 2018 -0800 @@ -51,13 +51,15 @@ -q: Quiet mode. No output at all. Ignored if -d is supplied. -d: Debug: Show incoming and parsed data. -v: Display version number. + -T: Change the destination table name from the default 'netdata'. + -t: Alter the maximum time (in ms) an open socket waits for data. Default: 500ms. -h: Help. You're lookin' at it. The default connection string is: "host=localhost port=5432 dbname=netdata user=netdata application_name=netdata-tsrelay" """ INSERT_SQL = """ - INSERT INTO netdata + INSERT INTO $1 ( time, host, metrics ) VALUES ( 'epoch'::timestamptz + ? * '1 second'::interval, ?, ? ) @@ -67,10 +69,13 @@ type Config = object of RootObj dbopts: string # The postgresql connection parameters. (See https://www.postgresql.org/docs/current/static/libpq-connect.html) - listen_port: int # The port to listen for incoming connections + dbtable: string # The name of the table to write to. + listen_port: int # The port to listen for incoming connections. listen_addr: string # The IP address listen for incoming connections. Defaults to inaddr_any. verbose: bool # Be informative debug: bool # Spew out raw data + insertsql: string # The SQL insert string after interpolating the table name. + timeout: int # How long to block, waiting on connection data. # Global configuration var conf: Config @@ -91,10 +96,10 @@ ## line and wait for stream timeout to determine a "sample". var buf: string = nil try: - result = client.recv_line( timeout=500 ) + result = client.recv_line( timeout=conf.timeout ) if result != "" and not result.is_nil: result = result & "\n" while buf != "": - buf = client.recv_line( timeout=500 ) + buf = client.recv_line( timeout=conf.timeout ) if buf != "" and not buf.is_nil: result = result & buf & "\n" except TimeoutError: discard @@ -111,7 +116,7 @@ for sample in split_lines( data ): if sample == "" or sample.is_nil: continue - #if conf.debug: echo sample.hl( fgBlack, bright=true ) + if conf.debug: echo sample.hl( fgBlack, bright=true ) var parsed: JsonNode try: @@ -158,7 +163,7 @@ host = sample[ "hostname" ].get_str sample.delete( "timestamp" ) sample.delete( "hostname" ) - db.exec sql( INSERT_SQL ), timestamp, host, sample + db.exec sql( conf.insertsql ), timestamp, host, sample db.exec sql( "COMMIT" ) except: let @@ -255,10 +260,13 @@ # result = Config( dbopts: "host=localhost port=5432 dbname=netdata user=netdata application_name=netdata-tsrelay", + dbtable: "netdata", listen_port: 14866, listen_addr: "0.0.0.0", verbose: true, - debug: false + debug: false, + timeout: 500, + insertsql: INSERT_SQL % [ "netdata" ] ) # always set debug mode if development build. @@ -281,12 +289,17 @@ of "quiet", "q": result.verbose = false - + of "version", "v": echo hl( "netdata_tsrelay " & VERSION, fgWhite, bright=true ) quit( 0 ) - + + of "timeout", "t": result.timeout = val.parse_int + + of "dbtable", "T": + result.insertsql = INSERT_SQL % [ val ] of "dbopts": result.dbopts = val + of "listen-addr", "a": result.listen_addr = val of "listen-port", "p": result.listen_port = val.parse_int