Multiple changes:
authorMahlon E. Smith <mahlon@laika.com>
Mon, 19 Feb 2018 18:22:25 -0800
changeset 11 475c9942eb15
parent 10 252cdb26f76b
child 12 450df27eaeec
Multiple changes: - Make the 'release' build the default. - Add a configurable socket timeout parameter. - Make the table name configurable. - Add usage docs to the README.
Makefile
README.md
netdata_tsrelay.nim
--- a/Makefile	Sun Feb 18 23:40:34 2018 -0800
+++ b/Makefile	Mon Feb 19 18:22:25 2018 -0800
@@ -1,7 +1,7 @@
 
 FILES = netdata_tsrelay.nim
 
-default: development
+default: release
 
 debug: ${FILES}
 	nim --assertions:on --nimcache:.cache c ${FILES}
--- a/README.md	Sun Feb 18 23:40:34 2018 -0800
+++ b/README.md	Mon Feb 19 18:22:25 2018 -0800
@@ -14,18 +14,17 @@
 Installation
 ------------
 
-You'll need a working [Nim](http://nim-lang.org) build environment to
-create the binary.
+You'll need a working [Nim](http://nim-lang.org) build environment and
+PostgreSQL development headers to compile the binary.
 
-Simply run `make release` to produce the binary.  Put it wherever you
-please.
+Simply run `make` to build it.  Put it wherever you please.
 
 
 Configuration
 -------------
 
 There are a few assumptions that should be satisfied before running
-this.
+this successfully.
 
 ### Database setup
 
@@ -39,25 +38,28 @@
 );
 ```
 
-Index it however you please based on how you intend to query the data,
-including JSON functional indexing, etc.  See PostgreSQL documentation
-for details.
+Index it based on how you intend to query the data, including JSON
+functional indexing, etc.  See PostgreSQL documentation for details.
 
 Strongly encouraged:  Promote this table to a Timescale "hypertable".
-See Timescale docs for that, but a quick example to partition
-automatically at weekly boundaries would look something like:
+See [Timescale](http://timescale.com) docs for that, but a quick example
+to partition automatically at weekly boundaries would look something
+like:
 
 ```sql
 SELECT create_hypertable( 'netdata', 'time', chunk_time_interval => 604800000000 );
 ```
 
+Timescale also has some great examples and advice for efficient [JSON
+indexing](http://docs.timescale.com/v0.8/using-timescaledb/schema-management#json)
+and queries.
 
 
 ### Netdata
 
 You'll likely want to pare down what netdata is sending.  Here's an
-example configuration for `netdata.conf` -- you will want to season
-this to taste.
+example configuration for `netdata.conf` -- season this to taste (what
+charts to send and frequency.)
 
 ```
 [backend]
@@ -67,8 +69,66 @@
     data source        = average
     destination        = machine-where-netdata-tsrelay-lives:14866
     prefix             = n
-    update every       = 10
-    buffer on failures = 6
+    update every       = 60
+    buffer on failures = 5
     send charts matching = !cpu.cpu* !ipv6* !users* nfs.rpc net.* net_drops.* net_packets.* !system.interrupts* system.* disk.* disk_space.* disk_ops.* mem.*
 ```
 
+
+Running the Relay
+-----------------
+
+### Options
+
+  * [-q|--quiet]:    Quiet mode.  No output at all. Ignored if -d is supplied.
+  * [-d|--debug]:    Debug mode.  Show incoming data.
+  * [--dbopts]:      PostgreSQL connection information.  (See below for more details.)
+  * [-h|--help]:     Display quick help text.
+  * [--listen-addr]: A specific IP address to listen on.  Defaults to INADDR_ANY.
+  * [--listen-port]: The port to listen for netdata JSON streams.
+                     Default is 14866.
+  * [-T|--dbtable]:  Change the table name to insert to.  Defaults to **netdata**.
+  * [-t|--timeout]:  Maximum time in milliseconds to wait for data.  Slow
+                     connections may need to increase this from the default 500 ms.
+  * [-v|--version]:  Show version.
+
+
+**Notes**
+
+Nim option parsing might be slightly different than what you're used to.
+Flags that require arguments must include an '=' or ':' character.
+
+  * --timeout=1000  *valid*
+  * --timeout:1000  *valid*
+  * --t:1000  *valid*
+  * --timeout 1000  *invalid*
+  * -t 1000  *invalid*
+
+All database connection options are passed as a key/val string to the
+*dbopts* flag.  The default is:
+
+	"host=localhost port=5432 dbname=netdata user=netdata application_name=netdata-tsrelay"
+
+Reference
+https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-
+PARAMKEYWORDS for all available options (including how to store
+passwords in a seperate file, enable SSL mode, etc.)
+
+
+### Daemonizing
+
+Use a tool of your choice to run this at system
+startup in the background.  My personal preference is
+[daemontools](https://cr.yp.to/daemontools.html), but I won't judge you
+if you use something else.
+
+Here's an example using the simple
+[daemon](https://www.freebsd.org/cgi/man.cgi?query=daemon&apropos=0&sektion=8&manpath=FreeBSD+11.0-RELEASE+and+Ports&arch=default&format=html) wrapper tool:
+
+	# daemon \
+		-o /var/log/netdata_tsrelay.log \
+		-p /var/run/netdata_tsrelay.pid \
+		-u nobody -cr \
+		/usr/local/bin/netdata_tsrelay \
+			--dbopts="dbname=metrics user=metrics host=db-master port=6432 application_name=netdata-tsrelay"
+
--- a/netdata_tsrelay.nim	Sun Feb 18 23:40:34 2018 -0800
+++ b/netdata_tsrelay.nim	Mon Feb 19 18:22:25 2018 -0800
@@ -51,13 +51,15 @@
   -q: Quiet mode.  No output at all.  Ignored if -d is supplied.
   -d: Debug: Show incoming and parsed data.
   -v: Display version number.
+  -T: Change the destination table name from the default 'netdata'.
+  -t: Alter the maximum time (in ms) an open socket waits for data.  Default: 500ms.
   -h: Help.  You're lookin' at it.
 
 The default connection string is:
   "host=localhost port=5432 dbname=netdata user=netdata application_name=netdata-tsrelay"
     """
     INSERT_SQL = """
-    INSERT INTO netdata
+    INSERT INTO $1
         ( time, host, metrics )
     VALUES
         ( 'epoch'::timestamptz + ? * '1 second'::interval, ?, ? )
@@ -67,10 +69,13 @@
 type
     Config = object of RootObj
         dbopts:      string  # The postgresql connection parameters.  (See https://www.postgresql.org/docs/current/static/libpq-connect.html)
-        listen_port: int     # The port to listen for incoming connections
+        dbtable:     string  # The name of the table to write to.
+        listen_port: int     # The port to listen for incoming connections.
         listen_addr: string  # The IP address listen for incoming connections.  Defaults to inaddr_any.
         verbose:     bool    # Be informative
         debug:       bool    # Spew out raw data
+        insertsql:   string  # The SQL insert string after interpolating the table name.
+        timeout:     int     # How long to block, waiting on connection data.
 
 # Global configuration
 var conf: Config
@@ -91,10 +96,10 @@
     ## line and wait for stream timeout to determine a "sample".
     var buf: string = nil
     try:
-        result = client.recv_line( timeout=500 )
+        result = client.recv_line( timeout=conf.timeout )
         if result != "" and not result.is_nil: result = result & "\n"
         while buf != "":
-            buf = client.recv_line( timeout=500 )
+            buf = client.recv_line( timeout=conf.timeout )
             if buf != "" and not buf.is_nil: result = result & buf & "\n"
     except TimeoutError:
         discard
@@ -111,7 +116,7 @@
 
     for sample in split_lines( data ):
         if sample == "" or sample.is_nil: continue
-        #if conf.debug: echo sample.hl( fgBlack, bright=true )
+        if conf.debug: echo sample.hl( fgBlack, bright=true )
 
         var parsed: JsonNode
         try:
@@ -158,7 +163,7 @@
                 host = sample[ "hostname" ].get_str
             sample.delete( "timestamp" )
             sample.delete( "hostname" )
-            db.exec sql( INSERT_SQL ), timestamp, host, sample
+            db.exec sql( conf.insertsql ), timestamp, host, sample
         db.exec sql( "COMMIT" )
     except:
         let
@@ -255,10 +260,13 @@
     #
     result = Config(
         dbopts: "host=localhost port=5432 dbname=netdata user=netdata application_name=netdata-tsrelay",
+        dbtable: "netdata",
         listen_port: 14866,
         listen_addr: "0.0.0.0",
         verbose: true,
-        debug: false
+        debug: false,
+        timeout: 500,
+        insertsql: INSERT_SQL % [ "netdata" ]
     )
 
     # always set debug mode if development build.
@@ -281,12 +289,17 @@
 
                 of "quiet", "q":
                     result.verbose = false
-            
+
                 of "version", "v":
                     echo hl( "netdata_tsrelay " & VERSION, fgWhite, bright=true )
                     quit( 0 )
-               
+
+                of "timeout", "t": result.timeout = val.parse_int
+
+                of "dbtable", "T":
+                    result.insertsql = INSERT_SQL % [ val ]
                 of "dbopts": result.dbopts = val
+
                 of "listen-addr", "a": result.listen_addr = val
                 of "listen-port", "p": result.listen_port = val.parse_int