This script can be used in NR Synthetics to copy Server Metrics into Insights Events.
/** API SETUP - remove this section to run in New Relic Synthetics **/
if ($http == null) { var $http = require('request'); }
/** API SETUP - remove this section to run in New Relic Synthetics **/
// Global variables we'll need
var config = {
'SCRIPTNAME': 'apm-server-to-insights',
'VERSION': '1.0.2',
'EVENT_NAME': 'ServerData',
'ACCOUNT_ID': 'xxx',
'QUERY_KEY_INSIGHTS': 'xxx',
'INSERT_KEY_INSIGHTS': 'xxx',
'REST_API_KEY': 'xxx'
};
var metricArr = [
{title: 'CPU System', metric: 'System/CPU/System/percent', value: 'average_value'},
{title: 'CPU User', metric: 'System/CPU/User/percent', value: 'average_value'},
{title: 'Memory Used', metric: 'System/Memory/Used/bytes', value: 'average_value'},
{title: 'Swap Used', metric: 'System/Swap/Used/bytes', value: 'average_value'},
{title: 'Disk Utilization', metric: 'System/Disk/All/Utilization/percent', value: ''},
{title: 'Network Received', metric: 'System/Network/All/Received/bytes/sec', value: 'total'},
{title: 'Network Transmitted', metric: 'System/Network/All/Transmitted/bytes/sec', value: 'total'} ];
// Get just the metric names here
var metricNamesArr = [];
for (var i=0; i < metricArr.length; i++) {
metricNamesArr.push( metricArr[i].metric );
}
// Initialize the internal objects
var metricData = {};
var maxTime = {};
var totalServerCount = 0;
var currServerCount = 0;
// Insert the collected data for this server into Insights
function insertServerData(serverId) {
var server = metricData[serverId];
var eventArr = [];
for (timestamp in server.events) {
eventArr.push(server.events[timestamp]);
}
// Setup the Insights insert options
var options = {
uri: 'https://insights-collector.newrelic.com/v1/accounts/' + config.ACCOUNT_ID + '/events',
headers: {'X-Insert-Key': config.INSERT_KEY_INSIGHTS},
json: true,
body: eventArr
};
if (eventArr.length > 0) {
$http.post(options, function(error, response, body) {
if (!error && response.statusCode == 200) {
console.log('Posted ' + eventArr.length + ' events for ' + server.name);
} else {
console.log('Post to Insights error! ' + server.name);
if (error) {
console.log(error);
} else {
console.log(body);
}
}
});
}
}
// This will pivot the data into consolidated timeslice events
function pivotMetricData(serverId) {
var server = metricData[serverId];
var serverName = server.name;
var serverMaxTime = maxTime[serverId];
var events = {};
server.events = events;
// console.log('Processing server: ' + serverName + ' max time = ' + serverMaxTime);
var metric_data = server.metric_data;
if (metric_data != null) {
var metrics = metric_data.metrics;
// Loop through the metrics
for (var i = 0; i < metrics.length; i++) {
var metricName = metrics[i].name;
var timeslices = metrics[i].timeslices;
// Loop through the timeslices for this metric
for (var j = 0; j < timeslices.length; j++) {
var slice = timeslices[j];
var timestamp = new Date(slice.to).getTime();
// If the time is good, add this value to our map
if (serverMaxTime < timestamp) {
var iEvent = events[timestamp];
if (iEvent == undefined) {
events[timestamp] = iEvent = {};
iEvent['eventType'] = config.EVENT_NAME;
iEvent['timestamp'] = timestamp;
iEvent['Server'] = iEvent['host'] = serverName;
iEvent['id'] = serverId;
}
var metricIndex = getMetricIndex(metricName);
var title = metricArr[metricIndex].title;
var valueIndex = metricArr[metricIndex].value;
var value = slice.values[valueIndex];
iEvent[title] = value;
}
} // for timeslices
} // for metrics
// Now that the pivot is complete, publish the data
insertServerData(serverId);
} else {
console.log(serverName + ' is skipped for some reason.');
} // if metric_data not null
}
function getMetricIndex(metricName) {
for (var i=0; i < metricArr.length; i++) {
var metric = metricArr[i];
if (metric.metric == metricName) {
return i;
}
}
return null;
}
// This will query the metrics for a specific server
function queryServer(serverId) {
var server = metricData[serverId];
// Make the HTTP Request options
var metricOpts = {
uri: 'https://api.newrelic.com/v2/servers/' + serverId + '/metrics/data.json',
headers: {'Accept': 'application/json', 'X-Api-Key': config.REST_API_KEY},
qsStringifyOptions: { arrayFormat: 'brackets' },
qs: { 'names': metricNamesArr },
json: true
};
// Request the metrics for this server
$http.get(metricOpts, function(error, response, body) {
currServerCount++;
if (!error && response.statusCode == 200) {
// Store the values in the metricData object
metricData[serverId].metric_data = body.metric_data;
pivotMetricData(serverId);
} else {
console.log('Query server error! ' + server.name);
if (error) {
console.log(error);
} else {
console.log(body);
}
}
});
}
// This will get all the servers for this page of results
// Once we get the last page of results then query metrics for each server
function getServers(pageNum) {
var serverListOpts = {
uri: 'https://api.newrelic.com/v2/servers.json?page=' + pageNum,
headers: {'Accept': 'application/json', 'X-Api-Key': config.REST_API_KEY},
json: true
};
$http.get(serverListOpts, function (error, response, body) {
if (!error && response.statusCode == 200) {
servers = body.servers;
console.log('Inside server count: ' + servers.length);
// Add the servers to the metricData object
for (var i=0; i < servers.length; i++) {
var server = servers[i];
metricData[server.id] = server;
// Initialize maxTime if we never saw this server before
if (maxTime[server.id] == null) {
maxTime[server.id] = 0;
}
}
// If there are 200 servers there may be more pages of data
if(servers.length == 200) {
// Get the next page of servers
getServers(pageNum + 1);
} else {
// We have all pages of servers, now query them
totalServerCount = Object.keys(metricData).length;
console.log('Got all server names: ' + totalServerCount);
// Query all the servers
for (serverId in metricData) {
queryServer(serverId);
// break; // Uncomment to get just 1 server
}
}
} else {
console.log('Query server list error on page ' + pageNum);
if (error) {
console.log(error);
} else {
console.log(body);
}
}
});
}
// Helper function to get the largest timestamp from the Insights data for a given server
function getMaxTime() {
var nrql = 'SELECT max(timestamp) FROM ' + config.EVENT_NAME + ' FACET id LIMIT 1000';
var maxTimeOpts = {
method: 'GET',
uri: 'https://insights-api.newrelic.com/v1/accounts/' + config.ACCOUNT_ID + '/query',
headers: {'Accept': 'application/json', 'X-Query-Key': config.QUERY_KEY_INSIGHTS},
qs: {'nrql': nrql},
json: true
};
// Call the Insights endpoint
$http.get(maxTimeOpts, function(error, response, body) {
if (!error && response.statusCode == 200) {
var facets = body.facets;
for (var i=0; i < facets.length; i++) {
var facet = facets[i];
maxTime[facet.name] = facet.results[0].max;
}
console.log('Set maxTime for ' + Object.keys(maxTime).length + ' server ids.');
} else {
console.log('Query Max Time error!');
if (error) {
console.log(error);
} else {
console.log(body);
}
}
// Now we can start getting server data
getServers(1);
});
}
console.log('Starting ' + config.SCRIPTNAME + ' version: ' + config.VERSION);
getMaxTime();