Shoora
4/29/2019 - 2:33 PM
fetch-google.js
// Requirements
var utils = require('./utils'),
async = require('async'),
googleapis = require('googleapis'),
GoogleModel = require('../models/base'),
config = require('../config'),
OAuth2Client = googleapis.OAuth2Client;
// Batch method to get data from analytics
function getAnalytics (access_token, refresh_token, username) {
// Create the date
// Params for get informations
var date = new Date(),
end_date = date.toISOString().slice(0,10); // today
date.setDate(date.getDate() -1); // Set date to yesterday
var start_date = date.toISOString().slice(0,10); // yesterday
// Instance the google outh client options
var oauth2Client = new OAuth2Client(config.GOOGLE_APP_ID, config.GOOGLE_APP_SECRET, config.AUTH_RETURN_BASE_URL + "/google");
// Add credentials to request client
oauth2Client.credentials = {
access_token: access_token,
refresh_token: refresh_token
};
// Call the api to get user informations
// Into discover put the first part of url that you can find in api explorer
googleapis
.discover('analytics', 'v3')
.execute(function(err, client) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred' + err);
return;
}
// get all profile
var request_profile = client.analytics.management.profiles.list({accountId: '~all', webPropertyId: '~all', fields: 'items(accountId,id,name)'}).withAuthClient(oauth2Client);
// Get the request results
request_profile.execute(function(err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred' + err);
return;
}
// Put the sitelist for the analytics profile in the database, create the class
var profileinfo = new GoogleModel("users");
profileinfo.update({username: username}, {$set: {analytics_accounts: result.items}});
// get webproperties id and get all data with a batch request
var profiles = result.items;
profiles.forEach(function (item) {
// Get all informations for the profile (start-date and end-date must be in '' because javascript has problems with - )
// Request for visit
var request_visits = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:visits"}).withAuthClient(oauth2Client);
// Request for new visit
var request_newvisits = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:newvisits"}).withAuthClient(oauth2Client);
// Request for returned visitors
var request_returnedvisits = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:visits", dimensions: "ga:dayssincelastvisit"}).withAuthClient(oauth2Client);
// Request for referral path
var request_referralpath = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:visits", dimensions: "ga:referralpath"}).withAuthClient(oauth2Client);
// Request for organicsearches
var request_organicsearches = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:organicsearches"}).withAuthClient(oauth2Client);
// Request for new visit
var request_source = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:visits", dimensions: "ga:source"}).withAuthClient(oauth2Client);
// Request for device
var request_device = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:visits", dimensions: "ga:devicecategory"}).withAuthClient(oauth2Client);
// Request for language
var request_language = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:visits", dimensions: "ga:language"}).withAuthClient(oauth2Client);
// Request for country
var request_country = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:visits", dimensions: "ga:country"}).withAuthClient(oauth2Client);
// Request for unique visit
var request_unique_visits = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:visitors"}).withAuthClient(oauth2Client);
// Request for unique visit referralPath
var request_unique_visits_referral = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:visitors", dimensions: "ga:referralpath"}).withAuthClient(oauth2Client);
// Request for unique visit organic searches
var request_unique_visits_organic = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:visitors,ga:organicsearches"}).withAuthClient(oauth2Client);
// Request for unique visit source
var request_unique_visits_source = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:visitors", dimensions: "ga:source"}).withAuthClient(oauth2Client);
// Request for unique visit device
var request_unique_visits_device = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:visitors", dimensions: "ga:devicecategory"}).withAuthClient(oauth2Client);
// Request for pageviews
var request_pageviews = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:pageviews"}).withAuthClient(oauth2Client);
// Request for pageviews by path
var request_pageviews_by_path = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:pageviews", dimensions: "ga:pagepath"}).withAuthClient(oauth2Client);
// Request for pageviews by path returned visitors
var request_pageviews_by_path_returned = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:pageviews", dimensions: "ga:pagepath,ga:dayssincelastvisit"}).withAuthClient(oauth2Client);
// Request for pageviews for visit
var request_pageviews_visit = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:pageviewspervisit"}).withAuthClient(oauth2Client);
// Request for pageviews for visit new
var request_pageviews_visit_new = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:pageviewspervisit,ga:newvisits"}).withAuthClient(oauth2Client);
// Request for returned visitors
var request_pageviews_returned = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:bounces,ga:uniquepageviews", dimensions: "ga:pagepath"}).withAuthClient(oauth2Client);
// Request for avg time on site
var request_avgtimeonsite = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:avgtimeonsite"}).withAuthClient(oauth2Client);
// Request for avg time on site for new visitors
var request_avgtimeonsite_new_visitors = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:avgtimeonsite,ga:newvisits"}).withAuthClient(oauth2Client);
// Request for avg time on site bounces
var request_avgtimeonsite_bounce = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:avgtimeonsite,ga:bounces"}).withAuthClient(oauth2Client);
// Request for bounces
var request_bounce = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:bounces"}).withAuthClient(oauth2Client);
// Request for bounces for new visits
var request_bounce_new_visitors = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:bounces,ga:newvisits"}).withAuthClient(oauth2Client);
// Request for bounces for new visits returned
var request_bounce_new_visitors_returned = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:bounces,ga:visits", dimensions: "ga:dayssincelastvisit"}).withAuthClient(oauth2Client);
// Request for new visits percent
var request_percentnewvisits = client.analytics.data.ga.get({ids: 'ga:' + item.id, 'start-date': start_date, 'end-date': end_date, metrics: "ga:percentnewvisits"}).withAuthClient(oauth2Client);
// Use async because with api batch function there are limit for number of request/second
// maybe it could be usefull parallelLimit() instead of series()
async.series([
function (callback) {
request_visits.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_newvisits.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_returnedvisits.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_referralpath.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_organicsearches.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_source.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_device.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_language.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_country.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_unique_visits.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_unique_visits_referral.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_unique_visits_organic.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_unique_visits_source.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_unique_visits_device.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_pageviews.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_pageviews_by_path.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_pageviews_by_path_returned.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_pageviews_visit.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_pageviews_visit_new.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_pageviews_returned.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_avgtimeonsite.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_avgtimeonsite_new_visitors.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_avgtimeonsite_bounce.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_bounce.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_bounce_new_visitors.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_bounce_new_visitors_returned.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
},
function (callback) {
request_percentnewvisits.execute( function (err, result) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
callback(null);
} else {
callback(null, result);
}
});
}
], function (err, results) {
if (err) {
utils.applog('error', 'From: fetch-google-batch.js error occurred ' + JSON.stringify(err));
return;
}
// Create the data and the value for collection
var date = new Date();
// for each request result get informations and add them to value
for (var i = 0; i < results.length; i++) {
// Only if there are results
if (results[i]) {
var value = {
username: username,
date: date,
};
value.profile_id = results[i].profileInfo.profileId;
value.profile_name = results[i].profileInfo.profileName;
value.account_id = results[i].profileInfo.accountId;
var field_name = results[i].query.metrics.join();
// If dimensions are set get all value
if (results[i].query.dimensions) {
// Create the reference name for field
field_name = field_name + "," + results[i].query.dimensions;
}
value.field_name = field_name;
value.columnHeaders = results[i].columnHeaders;
value.resultRows = results[i].rows;
// Store the data on database
var googlestore = new GoogleModel("analytics");
googlestore.insert(value, function (results) {
if (!results) {
utils.applog('error', 'Problem with get google data for: ' + username);
}
});
}
}
});
});
});
});
}
module.exports = {
getAnalytics: getAnalytics
};