checkaayush
6/11/2016 - 11:07 AM

Node.js script to load .txt files from a set of URLs asynchronously and combine results in order of URLs using default modules (and with non

Node.js script to load .txt files from a set of URLs asynchronously and combine results in order of URLs using default modules (and with non-default modules)

// Task using default Node.js packages

var http = require('http');

var urls = ['http://screeningtest.vdocipher.com/file1.txt',
            'http://screeningtest.vdocipher.com/file2.txt',
            'http://screeningtest.vdocipher.com/file3.txt',
            'http://screeningtest.vdocipher.com/file4.txt',
            'http://screeningtest.vdocipher.com/file5.txt',
            'http://screeningtest.vdocipher.com/file6.txt',
            'http://screeningtest.vdocipher.com/file7.txt',
            'http://screeningtest.vdocipher.com/file8.txt',
            'http://screeningtest.vdocipher.com/file9.txt',
            'http://screeningtest.vdocipher.com/file10.txt'];
            
var responses = [];
var completedRequests = 0;

/**
 * Given a 'url', it returns a JSON object of the form 
 * {"url": <someUrl>, body: <textFileContentAtUrl>}
 */
function httpGet(url, callback) {
  var returnVal = {url: url};
  http.get(url, function(response) {
    var output = '';
    response.on('data', function(chunk){
      output += chunk;
    });
    response.on('end', function(){
      returnVal['body'] = output.trim();
      callback(returnVal);            
    });
  });
}

function httpHelper(urls, callback) {
  for (var index = 0; index < urls.length; index++) {
    httpGet(urls[index], function(body) {
      completedRequests++;
      responses.push(body);

      if (completedRequests == urls.length) {
        callback(responses);
      }
    });
  }  
}

httpHelper(urls, function(responses) {
  var outputArray = [];
  for (var index in responses) {
    var urlIndex = urls.indexOf(responses[index].url);
    outputArray.splice(urlIndex, 0, responses[index].body);
  }
  var outputString = outputArray.join();
  console.log(outputString);
});


// Additional Method using non-default Node.js packages: 'async' & 'request'

var async = require('async');
var request = require('request');

function httpGet(url, callback) {
  const options = {
    url :  url,
    json : true
  };
  request(options,
    function(err, res, body) {
      callback(err, body);
    }
  );
}

async.map(urls, httpGet, function (err, res){
  if (err) return console.log(err);
  console.log(res.join());
});