Multiple changes:
- Make the 'release' build the default.
- Add a configurable socket timeout parameter.
- Make the table name configurable.
- Add usage docs to the README.
--- a/Makefile Sun Feb 18 23:40:34 2018 -0800
+++ b/Makefile Mon Feb 19 18:22:25 2018 -0800
@@ -1,7 +1,7 @@
FILES = netdata_tsrelay.nim
-default: development
+default: release
debug: ${FILES}
nim --assertions:on --nimcache:.cache c ${FILES}
--- a/README.md Sun Feb 18 23:40:34 2018 -0800
+++ b/README.md Mon Feb 19 18:22:25 2018 -0800
@@ -14,18 +14,17 @@
Installation
------------
-You'll need a working [Nim](http://nim-lang.org) build environment to
-create the binary.
+You'll need a working [Nim](http://nim-lang.org) build environment and
+PostgreSQL development headers to compile the binary.
-Simply run `make release` to produce the binary. Put it wherever you
-please.
+Simply run `make` to build it. Put it wherever you please.
Configuration
-------------
There are a few assumptions that should be satisfied before running
-this.
+this successfully.
### Database setup
@@ -39,25 +38,28 @@
);
```
-Index it however you please based on how you intend to query the data,
-including JSON functional indexing, etc. See PostgreSQL documentation
-for details.
+Index it based on how you intend to query the data, including JSON
+functional indexing, etc. See PostgreSQL documentation for details.
Strongly encouraged: Promote this table to a Timescale "hypertable".
-See Timescale docs for that, but a quick example to partition
-automatically at weekly boundaries would look something like:
+See [Timescale](http://timescale.com) docs for that, but a quick example
+to partition automatically at weekly boundaries would look something
+like:
```sql
SELECT create_hypertable( 'netdata', 'time', chunk_time_interval => 604800000000 );
```
+Timescale also has some great examples and advice for efficient [JSON
+indexing](http://docs.timescale.com/v0.8/using-timescaledb/schema-management#json)
+and queries.
### Netdata
You'll likely want to pare down what netdata is sending. Here's an
-example configuration for `netdata.conf` -- you will want to season
-this to taste.
+example configuration for `netdata.conf` -- season this to taste (what
+charts to send and frequency.)
```
[backend]
@@ -67,8 +69,66 @@
data source = average
destination = machine-where-netdata-tsrelay-lives:14866
prefix = n
- update every = 10
- buffer on failures = 6
+ update every = 60
+ buffer on failures = 5
send charts matching = !cpu.cpu* !ipv6* !users* nfs.rpc net.* net_drops.* net_packets.* !system.interrupts* system.* disk.* disk_space.* disk_ops.* mem.*
```
+
+Running the Relay
+-----------------
+
+### Options
+
+ * [-q|--quiet]: Quiet mode. No output at all. Ignored if -d is supplied.
+ * [-d|--debug]: Debug mode. Show incoming data.
+ * [--dbopts]: PostgreSQL connection information. (See below for more details.)
+ * [-h|--help]: Display quick help text.
+ * [--listen-addr]: A specific IP address to listen on. Defaults to INADDR_ANY.
+ * [--listen-port]: The port to listen for netdata JSON streams.
+ Default is 14866.
+ * [-T|--dbtable]: Change the table name to insert to. Defaults to **netdata**.
+ * [-t|--timeout]: Maximum time in milliseconds to wait for data. Slow
+ connections may need to increase this from the default 500 ms.
+ * [-v|--version]: Show version.
+
+
+**Notes**
+
+Nim option parsing might be slightly different than what you're used to.
+Flags that require arguments must include an '=' or ':' character.
+
+ * --timeout=1000 *valid*
+ * --timeout:1000 *valid*
+ * --t:1000 *valid*
+ * --timeout 1000 *invalid*
+ * -t 1000 *invalid*
+
+All database connection options are passed as a key/val string to the
+*dbopts* flag. The default is:
+
+ "host=localhost port=5432 dbname=netdata user=netdata application_name=netdata-tsrelay"
+
+Reference
+https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-
+PARAMKEYWORDS for all available options (including how to store
+passwords in a seperate file, enable SSL mode, etc.)
+
+
+### Daemonizing
+
+Use a tool of your choice to run this at system
+startup in the background. My personal preference is
+[daemontools](https://cr.yp.to/daemontools.html), but I won't judge you
+if you use something else.
+
+Here's an example using the simple
+[daemon](https://www.freebsd.org/cgi/man.cgi?query=daemon&apropos=0&sektion=8&manpath=FreeBSD+11.0-RELEASE+and+Ports&arch=default&format=html) wrapper tool:
+
+ # daemon \
+ -o /var/log/netdata_tsrelay.log \
+ -p /var/run/netdata_tsrelay.pid \
+ -u nobody -cr \
+ /usr/local/bin/netdata_tsrelay \
+ --dbopts="dbname=metrics user=metrics host=db-master port=6432 application_name=netdata-tsrelay"
+
--- a/netdata_tsrelay.nim Sun Feb 18 23:40:34 2018 -0800
+++ b/netdata_tsrelay.nim Mon Feb 19 18:22:25 2018 -0800
@@ -51,13 +51,15 @@
-q: Quiet mode. No output at all. Ignored if -d is supplied.
-d: Debug: Show incoming and parsed data.
-v: Display version number.
+ -T: Change the destination table name from the default 'netdata'.
+ -t: Alter the maximum time (in ms) an open socket waits for data. Default: 500ms.
-h: Help. You're lookin' at it.
The default connection string is:
"host=localhost port=5432 dbname=netdata user=netdata application_name=netdata-tsrelay"
"""
INSERT_SQL = """
- INSERT INTO netdata
+ INSERT INTO $1
( time, host, metrics )
VALUES
( 'epoch'::timestamptz + ? * '1 second'::interval, ?, ? )
@@ -67,10 +69,13 @@
type
Config = object of RootObj
dbopts: string # The postgresql connection parameters. (See https://www.postgresql.org/docs/current/static/libpq-connect.html)
- listen_port: int # The port to listen for incoming connections
+ dbtable: string # The name of the table to write to.
+ listen_port: int # The port to listen for incoming connections.
listen_addr: string # The IP address listen for incoming connections. Defaults to inaddr_any.
verbose: bool # Be informative
debug: bool # Spew out raw data
+ insertsql: string # The SQL insert string after interpolating the table name.
+ timeout: int # How long to block, waiting on connection data.
# Global configuration
var conf: Config
@@ -91,10 +96,10 @@
## line and wait for stream timeout to determine a "sample".
var buf: string = nil
try:
- result = client.recv_line( timeout=500 )
+ result = client.recv_line( timeout=conf.timeout )
if result != "" and not result.is_nil: result = result & "\n"
while buf != "":
- buf = client.recv_line( timeout=500 )
+ buf = client.recv_line( timeout=conf.timeout )
if buf != "" and not buf.is_nil: result = result & buf & "\n"
except TimeoutError:
discard
@@ -111,7 +116,7 @@
for sample in split_lines( data ):
if sample == "" or sample.is_nil: continue
- #if conf.debug: echo sample.hl( fgBlack, bright=true )
+ if conf.debug: echo sample.hl( fgBlack, bright=true )
var parsed: JsonNode
try:
@@ -158,7 +163,7 @@
host = sample[ "hostname" ].get_str
sample.delete( "timestamp" )
sample.delete( "hostname" )
- db.exec sql( INSERT_SQL ), timestamp, host, sample
+ db.exec sql( conf.insertsql ), timestamp, host, sample
db.exec sql( "COMMIT" )
except:
let
@@ -255,10 +260,13 @@
#
result = Config(
dbopts: "host=localhost port=5432 dbname=netdata user=netdata application_name=netdata-tsrelay",
+ dbtable: "netdata",
listen_port: 14866,
listen_addr: "0.0.0.0",
verbose: true,
- debug: false
+ debug: false,
+ timeout: 500,
+ insertsql: INSERT_SQL % [ "netdata" ]
)
# always set debug mode if development build.
@@ -281,12 +289,17 @@
of "quiet", "q":
result.verbose = false
-
+
of "version", "v":
echo hl( "netdata_tsrelay " & VERSION, fgWhite, bright=true )
quit( 0 )
-
+
+ of "timeout", "t": result.timeout = val.parse_int
+
+ of "dbtable", "T":
+ result.insertsql = INSERT_SQL % [ val ]
of "dbopts": result.dbopts = val
+
of "listen-addr", "a": result.listen_addr = val
of "listen-port", "p": result.listen_port = val.parse_int