Cacher is the code snippet organizer for pro developers

# fork of https://github.com/adamdrake/twitterstreamtemplate

from threading import Thread
from queue import Queue

from twython import TwythonStreamer
from requests.exceptions import ChunkedEncodingError

import dataset, datetime


class TwitterStream(TwythonStreamer):

    def __init__(self, consumer_key, consumer_secret, token, token_secret, tqueue):
        self.tweet_queue = tqueue
        super(TwitterStream, self).__init__(consumer_key, consumer_secret, token, token_secret)

    def on_success(self, data):
        if 'text' in data:
            self.tweet_queue.put(data)

    def on_error(self, status_code, data):
        print(status_code)
        # Want to stop trying to get data because of the error?
        # Uncomment the next line!
        # self.disconnect()


def stream_tweets(tweets_queue):
    # Input your credentials below
    consumer_key = ''
    consumer_secret = ''
    token = ''
    token_secret = ''
    try:
        stream = TwitterStream(consumer_key, consumer_secret, token, token_secret, tweets_queue)
        stream.statuses.filter(track='ebola, mers, antivirals, bigpharma antivirals, bigpharma pandemic flu, bird flu, H1N1, H5N1, H7N9, infectious outbreak, oseltamivir, pandemic, pandemic flu, pandemic vaccine, relenza, tamiflu, zanamivir, vaccine, vaccines, vaccinated, vaccinates, vaccinate, vaccination, vaccinations', language='it,en,fr,de,es')
    except ChunkedEncodingError:
        # Sometimes the API sends back one byte less than expected which results in an exception in the
        # current version of the requests library
        stream_tweets(tweet_queue)


def process_tweets(tweets_queue):
    while True:
        data = tweets_queue.get()

        # Do something with the tweet

        obs = ['retweet_count','favorite_count','lang','text','created_at','id_str']
        tmp = dict([(i, data[i]) for i in obs if i in data])

        # save coordinates
        try:
            if data['coordinates']:
                tmp['coordinates'] = str(data['coordinates']['coordinates'])
        except KeyError:
            pass
        
        # save user id
        try:
            if data['user']:
                tmp['user'] = data['user']['id_str']
                tmp['username'] = data['user']['name']
        except KeyError:
            pass        

        # if RT, save user id and tweet id of the original tweet 
        try:
            if data['retweeted_status']:
                tmp['rtOf'] = data['retweeted_status']['id_str']
                tmp['rtFrom'] = data['retweeted_status']['user']['id_str']
        except KeyError:
            pass               

        # save mentions
        try:
            if data['entities']['user_mentions']:
                tmp['user_mentions'] = str([i['id_str'] for i in data['entities']['user_mentions']])
        except KeyError:
            pass

        # save hashtags
        try:
            if data['entities']['hashtags']:
                tmp['hashtags'] = str([i['text'] for i in data['entities']['hashtags']])
        except KeyError:
            pass

        # save urls
        try:
            if data['entities']['urls']:
                tmp['urls'] = str([i['expanded_url'] for i in data['entities']['urls']])
        except KeyError:
            pass 

        #print tmp
        
        table.insert(tmp)

        # Indicate that the above enqueued task is complete.
        tweets_queue.task_done()

if __name__ == '__main__':
    
    now = datetime.datetime.today().ctime().replace(' ','-')
    db = dataset.connect('sqlite:///tweets-'+now+'.db')
    table = db['tweet']

    tweet_queue = Queue()
    Thread(target=stream_tweets, args=(tweet_queue,), daemon=True).start()

    process_tweets(tweet_queue)

# -*- coding: utf-8 -*-

import datetime
import sqlite3

import dataset
import pykov
import networkx as nx

def getGraph(path_dbfile):
  """
  Example:
  path_dbfile = '/path/to/tweets-Fri-Sep--5-12:47:34-2014.db'
  """
  tweetsSet = set()
  G = nx.DiGraph()
  Q = {}
  # read the database
  db = dataset.connect('sqlite:///' + path_dbfile)
  tweets = db[u'tweet']
  # create the objects
  for tweet in tweets:
    # avoid clones
    if tweet['id_str'] not in tweetsSet:
      tweetsSet.add(tweet['id_str'])
      # if the tweet is a retweet add a link (user, original user)
      if tweet["rtFrom"]:
        G.add_edge(tweet['user'],tweet["rtFrom"])
        Q[(tweet['user'],tweet["rtFrom"])] = Q.get((tweet['user'],tweet["rtFrom"]), 0.) + 1
      # if the tweet contains mentions add a link (user, mentioned user)
      # only in the case user and mentioned user are not equal
      if tweet['user_mentions']:
        for user in decodeMentions(tweet['user_mentions']):
          #if userA RT userB, twitter API return that userA mentions userB. Weird.
          if (user != tweet["rtFrom"]) and (user != tweet['user']):
            G.add_edge(tweet['user'],user)
            Q[(tweet['user'],user)] = Q.get((tweet['user'],user), 0.) + 1 
  return G, Q

def decodeMentions(data):
  """
  """
  return [d.split("'")[1] for d in data.split(",")]

def getTransitionMatrix(G,Q):
  """
  """
  T = pykov.Chain()
  for edge in G.edges():
    T[edge] = Q[edge]
  T.stochastic()
  return T