Cacher is the code snippet organizer for pro developers

const request = require('request');
const _ = require('lodash');
const fetch = require('node-fetch');
const slackService = require('./slackService');

const APPNNEXUS_AUTH_URL = 'https://api.appnexus.com/auth';
const INVENTORY_LIST_ID = 11837;
const INVENTORY_LIST_NAME = 'NLO';
const APPNEXUS_INVENTORY_LIST_URL = `https://api.appnexus.com/inventory-list/${INVENTORY_LIST_ID}/item`;
const WHITELISTED_DOMAINS_FEED_URL = ['https://docs.google.com/spreadsheets/d/e/2PACX-1vSpmQIH3EJTDVkUi7lyoRcEJ7J6qMsItrNW_6YO9SfQITqe68-UGCZ0YfjnGgL_WH_Me1Ps0pGAdS3D/pub?output=csv', 'https://docs.google.com/spreadsheets/d/e/2PACX-1vSWGgd-O_h2Cz88CsMhq7W73JEZMcjZZJgPrHFeNw9MhO9d2ydDA-2jm_U5vJsJSK9ijv9C6C6Bh7n5/pub?output=csv'];

slackService.setSlackUrl(process.env.SLACK_URL);

const getAppNexusToken = () => new Promise((resolve, reject) => {
  request.post({
    headers: {
      'content-type': 'application/json',
    },
    url: APPNNEXUS_AUTH_URL,
    body: JSON.stringify({
      auth: {
        username: process.env.APPNEXUS_USERNAME,
        password: process.env.APPNEXUS_PASSWORD,
      },
    }),
  }, (error, response, body) => {
    const data = JSON.parse(body).response;
    if (data.status !== 'OK') return reject('Logging in to AppNexus failed.');
    resolve(data.token);
  });
});

const getAllCurrentInventoryListItems = (id, token) => new Promise((resolve, reject) => {
  const url = `${APPNEXUS_INVENTORY_LIST_URL}?num_elements=5000`.replace('(:INVENTORY_LIST_ID:)', id);
  request.get({
    headers: {
      'content-type': 'application/json',
      Authorization: token,
    },
    url,

  }, (error, response, body) => {
    const data = JSON.parse(body).response;
    if (data.status !== 'OK') return reject('Could not get current domains.');
    resolve(data['inventory-list-items']);
  });
});

const deleteInventoryListItems = (token, inventoryListItemsIds, id) => new Promise((resolve, reject) => {
  const url = `${APPNEXUS_INVENTORY_LIST_URL}?id=${_.uniq(inventoryListItemsIds).join`,`}`.replace('(:INVENTORY_LIST_ID:)', id);

  try {
    request.delete({
      headers: {
        'content-type': 'application/json',
        Authorization: token,
      },
      url,
    }, (error, response, body) => {
      const data = JSON.parse(body).response;
      if (data.status !== 'OK') return reject('Deleting items failed');
      resolve();
    });
  } catch (e) {
    console.log('Something went wrong.');
  }
});

const promiseSerial = funcs =>
  funcs.reduce(
    (promise, func) =>
      promise.then(result => func().then(Array.prototype.concat.bind(result))),
    Promise.resolve([]),
  );

const updateInventoryListItems = (token, items, id) => new Promise(async (resolve, reject) => {
  const url = `${APPNEXUS_INVENTORY_LIST_URL}`.replace('(:INVENTORY_LIST_ID:)', id);

  const groupedItems = [];
  const size = 100;

  while (items.length > 0) { groupedItems.push(items.splice(0, size)); }

  try {
    const promises = groupedItems.map(group => () => new Promise(async (resolve, reject) => {
      const uniqGroup = [];
      group.forEach((element) => {
        if (!uniqGroup.find(e => e.url === element.url)) {
          uniqGroup.push(element);
        }
      });
      const urls = [...uniqGroup];

      const data = await fetch(url, {
        method: 'POST',
        headers: {
          'content-type': 'application/json',
          Authorization: token,
        },
        body: JSON.stringify({
          'inventory-list-items': urls,
        }),
      });
      const { response } = await data.json();

      if (response.status !== 'OK') return reject(response);

      resolve(response);
    }));
    await promiseSerial(promises);

    resolve(items);
  } catch (e) {
    setTimeout(() => {
      console.log(e, 'Run after error');
      run(INVENTORY_LIST_ID, WHITELISTED_DOMAINS_FEED_URL);
    }, 2000);
  }
});


const getCsvs = async (urls) => {
  const promises = urls.map(url => new Promise((resolve, reject) => {
    request(url, async (error, response, body) => {
      try {
        if (error) throw new Error('Could not load feed.');

        const entries = body.split('\n')
          .slice(1)
          .map(e => e.replace('\r', ''))
          .map(e => e.replace(',', ''));

        return resolve(entries);
      } catch (e) { }
    });
  }));

  const res = await Promise.all(promises);
  const data = Array.prototype.concat.apply([], res);
  return data;
};

const run = async (id, urls, listName) => {
  const entries = await getCsvs(urls);
  const token = await getAppNexusToken();

  if (!entries || entries.length < 50) {
    throw new Error('Whitelist feed contains less than 50 domains. Looks like something is wrong!');
  }

  const currentInventoryListItems = await getAllCurrentInventoryListItems(id, token);

  const existingDomains = currentInventoryListItems.map(item => item.inventory_url);

  const newInventoryListItems = entries.map((url) => {
    let rewrittenUrl = url.replace('https://', '').replace('http://', '').trim();
    rewrittenUrl = rewrittenUrl.replace('www.', '');

    if (rewrittenUrl.substr(rewrittenUrl.length - 1, 1) === '/') return rewrittenUrl.toLowerCase().substr(0, rewrittenUrl.length - 1);
    return rewrittenUrl.toLowerCase();
  });

  const toAdd = newInventoryListItems; // .filter(u => !existingDomains.includes(u));

  const toBeDeleted = existingDomains.filter(e => !newInventoryListItems.map(b => b.toLowerCase()).includes(e.toLowerCase()));
  const idsToDelete = toBeDeleted.map(u => currentInventoryListItems.find(b => b.inventory_url === u).id);

  if (toAdd.length > 0) {
    const items = toAdd.map(e => e.replace(',', '')).map(item => ({
      url: item,
    })).filter(item => item.url);
    const originalItems = [...items];
    await updateInventoryListItems(token, items, id);
    console.log('Done adding', originalItems);
    if (originalItems.length > 0) {
      const urls = originalItems.map(u => u.url).join(', ');
      const message = `Uploaded new urls to inventory list ${id} for ${listName}: ${urls} `;
      slackService.sendMessageToSlack(message);
    }
  }

  if (idsToDelete.length > 0) {
    await deleteInventoryListItems(token, idsToDelete, id);
    console.log('Done removing', toBeDeleted);
    const urls = toBeDeleted.join(', ');
    const message = `Removed urls from inventory list ${id} for ${listName}: ${urls} `;
    slackService.sendMessageToSlack(message);
  }

  process.exit();
};

run(INVENTORY_LIST_ID, WHITELISTED_DOMAINS_FEED_URL, INVENTORY_LIST_NAME);

const request = require('request');


const sendMessageToSlack = text => new Promise((resolve) => {
  request.post({
    url: this.slackUrl,
    body: JSON.stringify({
      text,
    }),
  }, () => resolve());
});

const setSlackUrl = (url) => { this.slackUrl = url; };

const slackService = () => ({
  setSlackUrl,
  sendMessageToSlack,
});

module.exports = slackService();

{
  "name": "@meh/appnexus_inventory_list_syncer",
  "version": "1.0.0",
  "description": "MEH Appnexus whitelist domain syncer",
  "main": "src/server.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1",
    "sync": "node jobs/sync.js"
  },
  "engines": {
    "node": "10.15.1"
  },
  "repository": {
    "type": "git",
    "url": "git@gitlab.com:GreenhouseGroup/meh/appnexus_inventory_list_syncer.git"
  },
  "keywords": [],
  "author": "",
  "license": "ISC",
  "homepage": "https://gitlab.com/GreenhouseGroup/meh/appnexus_inventory_list_syncer",
  "dependencies": {
    "express": "^4.16.2",
    "lodash": "^4.17.11",
    "node-fetch": "^2.3.0",
    "request": "^2.88.0"
  }
}