mahlon/netdata-timescale-relay
1
0
Fork 0
Code Issues Pull requests Activity

Multiple changes:

Browse source 
- Make the 'release' build the default.
  - Add a configurable socket timeout parameter.
  - Make the table name configurable.
  - Add usage docs to the README.
This commit is contained in:
Mahlon E. Smith 2018-02-19 18:22:25 -08:00
parent 895453f4d3
commit b8d036a5da
3 changed files with 97 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Makefile
View file

@ -1,7 +1,7 @@
FILES = netdata_tsrelay.nim
default: development
default: release
debug: ${FILES}
nim --assertions:on --nimcache:.cache c ${FILES}

88
README.md
View file

@ -14,18 +14,17 @@ backed tables (though that's not technically a requirement.)
Installation
------------
You'll need a working [Nim](http://nim-lang.org) build environment to
create the binary.
You'll need a working [Nim](http://nim-lang.org) build environment and
PostgreSQL development headers to compile the binary.
Simply run `make release` to produce the binary. Put it wherever you
please.
Simply run `make` to build it. Put it wherever you please.
Configuration
-------------
There are a few assumptions that should be satisfied before running
this.
this successfully.
### Database setup
@ -39,25 +38,28 @@ CREATE TABLE netdata (
);
```
Index it however you please based on how you intend to query the data,
including JSON functional indexing, etc. See PostgreSQL documentation
for details.
Index it based on how you intend to query the data, including JSON
functional indexing, etc. See PostgreSQL documentation for details.
Strongly encouraged: Promote this table to a Timescale "hypertable".
See Timescale docs for that, but a quick example to partition
automatically at weekly boundaries would look something like:
See [Timescale](http://timescale.com) docs for that, but a quick example
to partition automatically at weekly boundaries would look something
like:
```sql
SELECT create_hypertable( 'netdata', 'time', chunk_time_interval => 604800000000 );
```
Timescale also has some great examples and advice for efficient [JSON
indexing](http://docs.timescale.com/v0.8/using-timescaledb/schema-management#json)
and queries.
### Netdata
You'll likely want to pare down what netdata is sending. Here's an
example configuration for `netdata.conf` -- you will want to season
this to taste.
example configuration for `netdata.conf` -- season this to taste (what
charts to send and frequency.)
```
[backend]
@ -67,8 +69,66 @@ this to taste.
data source = average
destination = machine-where-netdata-tsrelay-lives:14866
prefix = n
update every = 10
buffer on failures = 6
update every = 60
buffer on failures = 5
send charts matching = !cpu.cpu* !ipv6* !users* nfs.rpc net.* net_drops.* net_packets.* !system.interrupts* system.* disk.* disk_space.* disk_ops.* mem.*
```
Running the Relay
-----------------
### Options
* [-q|--quiet]: Quiet mode. No output at all. Ignored if -d is supplied.
* [-d|--debug]: Debug mode. Show incoming data.
* [--dbopts]: PostgreSQL connection information. (See below for more details.)
* [-h|--help]: Display quick help text.
* [--listen-addr]: A specific IP address to listen on. Defaults to INADDR_ANY.
* [--listen-port]: The port to listen for netdata JSON streams.
Default is 14866.
* [-T|--dbtable]: Change the table name to insert to. Defaults to **netdata**.
* [-t|--timeout]: Maximum time in milliseconds to wait for data. Slow
connections may need to increase this from the default 500 ms.
* [-v|--version]: Show version.
**Notes**
Nim option parsing might be slightly different than what you're used to.
Flags that require arguments must include an '=' or ':' character.
* --timeout=1000 *valid*
* --timeout:1000 *valid*
* --t:1000 *valid*
* --timeout 1000 *invalid*
* -t 1000 *invalid*
All database connection options are passed as a key/val string to the
*dbopts* flag. The default is:
"host=localhost port=5432 dbname=netdata user=netdata application_name=netdata-tsrelay"
Reference
https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-
PARAMKEYWORDS for all available options (including how to store
passwords in a seperate file, enable SSL mode, etc.)
### Daemonizing
Use a tool of your choice to run this at system
startup in the background. My personal preference is
[daemontools](https://cr.yp.to/daemontools.html), but I won't judge you
if you use something else.
Here's an example using the simple
[daemon](https://www.freebsd.org/cgi/man.cgi?query=daemon&apropos=0&sektion=8&manpath=FreeBSD+11.0-RELEASE+and+Ports&arch=default&format=html) wrapper tool:
# daemon \
-o /var/log/netdata_tsrelay.log \
-p /var/run/netdata_tsrelay.pid \
-u nobody -cr \
/usr/local/bin/netdata_tsrelay \
--dbopts="dbname=metrics user=metrics host=db-master port=6432 application_name=netdata-tsrelay"

27
netdata_tsrelay.nim
View file

@ -51,13 +51,15 @@ const
-q: Quiet mode. No output at all. Ignored if -d is supplied.
-d: Debug: Show incoming and parsed data.
-v: Display version number.
-T: Change the destination table name from the default 'netdata'.
-t: Alter the maximum time (in ms) an open socket waits for data. Default: 500ms.
-h: Help. You're lookin' at it.
The default connection string is:
"host=localhost port=5432 dbname=netdata user=netdata application_name=netdata-tsrelay"
"""
INSERT_SQL = """
INSERT INTO netdata
INSERT INTO $1
( time, host, metrics )
VALUES
( 'epoch'::timestamptz + ? * '1 second'::interval, ?, ? )
@ -67,10 +69,13 @@ The default connection string is:
type
Config = object of RootObj
dbopts: string # The postgresql connection parameters. (See https://www.postgresql.org/docs/current/static/libpq-connect.html)
listen_port: int # The port to listen for incoming connections
dbtable: string # The name of the table to write to.
listen_port: int # The port to listen for incoming connections.
listen_addr: string # The IP address listen for incoming connections. Defaults to inaddr_any.
verbose: bool # Be informative
debug: bool # Spew out raw data
insertsql: string # The SQL insert string after interpolating the table name.
timeout: int # How long to block, waiting on connection data.
# Global configuration
var conf: Config
@ -91,10 +96,10 @@ proc fetch_data( client: Socket ): string =
## line and wait for stream timeout to determine a "sample".
var buf: string = nil
try:
result = client.recv_line( timeout=500 )
result = client.recv_line( timeout=conf.timeout )
if result != "" and not result.is_nil: result = result & "\n"
while buf != "":
buf = client.recv_line( timeout=500 )
buf = client.recv_line( timeout=conf.timeout )
if buf != "" and not buf.is_nil: result = result & buf & "\n"
except TimeoutError:
discard
@ -111,7 +116,7 @@ proc parse_data( data: string ): seq[ JsonNode ] =
for sample in split_lines( data ):
if sample == "" or sample.is_nil: continue
#if conf.debug: echo sample.hl( fgBlack, bright=true )
if conf.debug: echo sample.hl( fgBlack, bright=true )
var parsed: JsonNode
try:
@ -158,7 +163,7 @@ proc write_to_database( samples: seq[ JsonNode ] ): void =
host = sample[ "hostname" ].get_str
sample.delete( "timestamp" )
sample.delete( "hostname" )
db.exec sql( INSERT_SQL ), timestamp, host, sample
db.exec sql( conf.insertsql ), timestamp, host, sample
db.exec sql( "COMMIT" )
except:
let
@ -255,10 +260,13 @@ proc parse_cmdline: Config =
#
result = Config(
dbopts: "host=localhost port=5432 dbname=netdata user=netdata application_name=netdata-tsrelay",
dbtable: "netdata",
listen_port: 14866,
listen_addr: "0.0.0.0",
verbose: true,
debug: false
debug: false,
timeout: 500,
insertsql: INSERT_SQL % [ "netdata" ]
)
# always set debug mode if development build.
@ -286,7 +294,12 @@ proc parse_cmdline: Config =
echo hl( "netdata_tsrelay " & VERSION, fgWhite, bright=true )
quit( 0 )
of "timeout", "t": result.timeout = val.parse_int
of "dbtable", "T":
result.insertsql = INSERT_SQL % [ val ]
of "dbopts": result.dbopts = val
of "listen-addr", "a": result.listen_addr = val
of "listen-port", "p": result.listen_port = val.parse_int