Cacher is the code snippet organizer for pro developers

"""
Playing with Python's `Counter`

- it's like a dictionary
- values can be positive/negative integers
- keys correspond to the things you want to count

"""

>>> from collections import Counter
>>> c = Counter()  # Create a Counter
>>> c['widgets'] += 1  # start counting 'widgets'
>>> c
Counter({'widgets': 1})

# (most) regular dict methods are available
>>> c.keys()
['widgets']

>>> c.values()
[1]

>>> 'widgets' in c
True

# `update` will create new keys or adjust the counts for
# existing keys
>>> c.update({'foo': 1})
>>> c
Counter({'widgets': 1, 'foo': 1})

# calling `update` again will increment the value of 'foo'
>>> c.update({'foo': 1})
>>> c
Counter({'widgets': 1, 'foo': 2})

# You can create a Counter from an iterable
>>> c = Counter(['larry', 'moe', 'curly'])
>>> c
Counter({'larry': 1, 'curly': 1, 'moe': 1})

# Or you can pass in keyword args
>>> c = Counter(ravens=34, niners=31)
>>> c
Counter({'ravens': 34, 'niners': 31})

# `elements` gives you an iterator that yeilds a `key` for each
# `count`. (You can also create a counter from an iterable).
>>> colors = ['red', 'blue', 'yellow']
>>> c = Counter(colors)
>>> c
Counter({'blue': 1, 'yellow': 1, 'red': 1})

>>> c['red'] += 2  # Three 'red's
>>> c['blue'] += 1 # Two 'blues's
>>> c
Counter({'red': 3, 'blue': 2, 'yellow': 1})

>>> list(c.elements())
['blue', 'blue', 'yellow', 'red', 'red', 'red']

# Finding the N "most common" elements
>>> c.most_common(2)
[('red', 3), ('blue', 2)]

# Trick: Find the most common letters in a string:
>>> Counter('supercalifragilisticexpialidocious').most_common(3)
[('i', 7), ('a', 3), ('c', 3)]

# Subtracting counts
>>> money = {'gold': 1001, 'silver': 501, 'copper': 101}
>>> shield = {'gold': 25}
>>> sword = {'gold': 100, 'silver':50}

# initialize your bank
>>> c = Counter(money)
>>> c
Counter({'gold': 1001, 'silver': 501, 'copper': 101})

# Buy a shield
>>> c.subtract(shield)
>>> c
Counter({'gold': 976, 'silver': 501, 'copper': 101})

# Buy a sword
>>> c.subtract(sword)
Counter({'gold': 876, 'silver': 451, 'copper': 101})

# Buy a Castle!
>>> castle = {'gold': 50000, 'silver': 9999, 'copper': 350}
>>> c.subtract(castle)
>>> c
Counter({'copper': -249, 'silver': -9548, 'gold': -49124})
# oops!

# start over!
>>> c.clear()
Counter()

"""
Use a Counter to find the most common words in "The Wonderful Wizard of Oz" by
L. Frank Baum.

Available in plain text at:
https://ia700500.us.archive.org/2/items/thewonderfulwiza00055gut/wizoz10.txt

short link: http://bit.ly/thewonderfulwizard

Note: This code also counts the words in the header, so it's not a *realistic*
applicaton, but more of a demonstration of python's Counter.

Running this code should give you something like this:

    $ python count_words.py

    The Top 10 words
    the: 2808
    and: 1630
    to: 1143
    of: 869
    a: 819
    I: 597
    was: 502
    you: 486
    in: 476
    he: 408

"""

from collections import Counter
import re
import urllib  # for more pleasant http, use http://bit.ly/python-requests


def main(n=10):

    # Download the content
    content = urllib.urlopen('http://bit.ly/thewonderfulwizard').read()

    # Clean the content a little
    content = re.sub('\s+', ' ', content)  # condense all whitespace
    content = re.sub('[^A-Za-z ]+', '', content)  # remove non-alpha chars
    words = content.split()

    # Start counting
    word_count = Counter(words)

    # The Top-N words
    print("The Top {0} words".format(n))
    for word, count in word_count.most_common(n):
        print("{0}: {1}".format(word, count))


if __name__ == "__main__":
    main()

# Most popular suffixes

# Importing NLTK stuff
import nltk
from nltk import FreqDist
from nltk.book import *
from nltk.tokenize import *

def top_suffixes(words):
    # Taking each word in the text and converting it into lowercase
    words = [w.lower() for w in words]
    # I need a list where I can store the suffixes I get
    suffixes = []
    '''For each word, if it has more than 5 characters,
    I will get its last two characters and store the results in the empty list'''
    for word in words:
        if len(word) >= 5:				
            suffix = word[-2:]			
            suffixes.append(suffix)
    '''I define a variable with the frequencies for each suffix. Then the
    function will return the 10 most common ones'''
    fd = FreqDist(suffixes)
    return (fd.most_common(10))           


emma_words = nltk.corpus.gutenberg.words('austen-emma.txt')
print(top_suffixes(emma_words))

# Pruefungzum Kurs 'Einführung in die Progammierung mit Python'


import nltk.data                                        # To tokenize, delete punctuation, stopwords etc
from nltk.tokenize import RegexpTokenizer               # To use regular expressions and tokenize words
import os                                               # To be able to set the working directory and open function
import pylab as pl                                      # Needed to plot
import numpy as np                                      # Needed to plot
from collections import Counter                         # Needed in order to extract the frequency of the words and
                                                        # Use of the argument 'most_common()'
from nltk.corpus import wordnet                         # to use WordNet

# If I want the user to input the text himself
# datas = input("Insert your data: ")

# If I want the user to use a .txt file as corpus

os.chdir("/Users/MissOgra/Documents")
datas = input("Enter the file name with its extension: ")
try:
    datas = open(datas).read()

except:
    print("Are you sure that's the correct file name? - Also make sure it is in the Documents folder")
    exit()


# Tokenizing by sentences
def sentence(datas):
    tokenizer = nltk.data.load("tokenizers/punkt/PY3/english.pickle")
    sen = tokenizer.tokenize(datas)
    return str(sen)

# To eliminate punctuation and divide words
def nopunct(datas):
    from nltk.tokenize import RegexpTokenizer
    nopunct_a = RegexpTokenizer("[a-zA-Z]+")           # It will eliminate anything that is not a character from A to Z.
    return nopunct_a.tokenize(datas)                   # It will divide the input in words.

#Deleting stopwords
def stopwords(datas):
    from nltk.corpus import stopwords
    stop = stopwords.words('english')                  # Here I need to specify the .txt language file with the stopwords
    wordlist = datas
    filtered = [i for i in wordlist if i not in stop]
    return filtered


# Tokenisierung and more

os.chdir("/Users/MissOgra/Documents")                 # It sets the file's directory
corpus = open('prueba.txt', 'w+')                     # It creates and writes a file
sentences = sentence(datas)                           # The input will be divided into sentences
#sen_write = str(corpus.write(sentences))             # The output is written in the prueba.txt file if wanted.
nopuncts = str(nopunct(sentences))                    # The sentences will be divided into words without punctuation.
#nopuncts_write = str(corpus.write(nopuncts))         # The output is written in the prueba.txt file if wanted.
stops = str(stopwords(nopunct(sentences)))            # It deletes all the functional words
stopi = str(corpus.write(stops))                      # The output is written in the prueba.txt file if wanted.
corpus.close()                                        # It closes the file



# Counting words

korpi = open("prueba.txt")                             # filtered words

str_korpi = korpi.read().strip()[1:-1].split(", ")    # Read, delete square brackets from the list and split the string

# Now I need to count how many times a word appear in the text:


counts = dict()
for word in str_korpi:
    word = word.lower()                               # It's important to get all words in low cases,
    if word not in counts:                            # otherwise it will count "house" and "House" as different words
        counts[word] = 1
    else:
        counts[word] = counts[word] + 1

korpi.close()

c = Counter(counts)                                   # It counts the frequency of a word

# To write a file with the sorted values to get the most common first:

korpis = open('counted.txt', 'w+')
countip = str(c.most_common())
str(korpis.write(countip))
korpis.close()

# To get the top 5:

most_frequent = c.most_common(5)                      # It looks for the 5 most common
mosts_frequent = dict(most_frequent)                  # I need a dictionary to create the plot
print("-----")
print("And the top 5 most popular words:")
print("-----")
print(most_frequent)


# Creating the histogram:

X = np.arange(len(mosts_frequent))
pl.bar(X, mosts_frequent.values(), align='center', width=0.5, color='green')
pl.xticks(X, mosts_frequent.keys())
ymax = max(mosts_frequent.values()) + 1
pl.ylim(0, ymax)
pl.show()

# Now I want to get the definition of each word in the previous top 5 using WordNet:

# First step - Create a list that contains only the keys of the top 5 dict.

key_counts = mosts_frequent.keys()
key_list = list(key_counts)

# Since each word has extra double quotations, I need to delete them so the WordNet code can run

str1 = ''.join(key_list)                             # Extra double quotations deleted. Now each word has single quotes
string = str1.replace("'", " ")                      # Single quotations deleted. Now I just have a str line with words
lst_string = string.split()                          # Split gives a list as a result

# Now that I have a clean list, I can use it to create a file with the top 5 words + their definition

for word in lst_string:
    syn = wordnet.synsets(word)[0]
    print(word,':', syn.definition())



'''
Bibliography:

- Severance, Charles. Python for Informatics: Exploring information. Version 2.7.0.
- Perking, Jacob. Python 3 Text Processing with NLTK 3 Cookbook. 2014
- Histogram's code source: http://stackoverflow.com/questions/23834570/plotting-histogram-from-dictionary-python

'''

# Getting KWICs

search = "der"
for word in text:
	if word == search:
		indice = text.index(search)
		print(text[indice-1], text[indice], text[indice+1])