Cacher is the code snippet organizer for pro developers

# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging
import click
import math
from tqdm import tqdm
import numpy
from preshed.counter import PreshCounter

from spacy.compat import fix_text
from spacy.vectors import Vectors
from spacy.util import prints, ensure_path, get_lang_class


@click.command(name='init-model')
@click.argument('lang', type=str)
@click.argument('output_dir', type=click.Path())
@click.argument('freqs_loc', type=click.Path())
@click.option('--clusters_loc', '-c',
              help='location of brown clusters data',
              type=click.Path())
@click.option('--vectors_loc', '-v',
              help='location of vectors file in GenSim text format',
              type=click.Path())
@click.option('--binary_vectors', '-b',
              help='Flag indicating vectors file is in GenSim binary format',
              is_flag=True)
@click.option('--prune_vectors', '-V',
              help='number of vectors to prune to',
              default=-1,
              type=int)
def init_model(lang, output_dir, freqs_loc, clusters_loc=None,
               vectors_loc=None, binary_vectors=True, prune_vectors=-1):
    """
    Create a new model from raw data, like word frequencies, Brown clusters
    and word vectors.

    LANG: language code

    OUTPUT_DIR: model output directory

    FREQS_LOC: location of words frequencies file
    """
    freqs_loc = ensure_path(freqs_loc)
    clusters_loc = ensure_path(clusters_loc)
    vectors_loc = ensure_path(vectors_loc)
    output_dir = ensure_path(output_dir)

    if not freqs_loc.exists():
        prints(freqs_loc, title="Can't find words frequencies file", exits=1)

    probs, oov_prob = read_freqs(freqs_loc)

    if vectors_loc:
        if binary_vectors:
            vectors_data, vector_keys = read_binary_vectors(vectors_loc)
        else:
            vectors_data, vector_keys = read_vectors(vectors_loc)
    else:
        vectors_data = numpy.empty((0, 0))
        vector_keys = []

    clusters = read_clusters(clusters_loc) if clusters_loc else {}

    nlp = create_model(lang, probs, oov_prob, clusters, vectors_data,
                       vector_keys, prune_vectors)

    if not output_dir.exists():
        output_dir.mkdir()
    nlp.to_disk(output_dir)
    return nlp


def create_model(lang, probs, oov_prob, clusters, vectors_data, vector_keys,
                 prune_vectors):
    print("Creating model...")
    lang_class = get_lang_class(lang)
    nlp = lang_class()
    for lexeme in nlp.vocab:
        lexeme.rank = 0

    lex_added = 0
    for i, (word, prob) in enumerate(tqdm(
            sorted(probs.items(), key=lambda item: item[1], reverse=True))):
        lexeme = nlp.vocab[word]
        lexeme.rank = i
        lexeme.prob = prob
        lexeme.is_oov = False
        # Decode as a little-endian string, so that we can do & 15 to get
        # the first 4 bits. See _parse_features.pyx
        if word in clusters:
            lexeme.cluster = int(clusters[word][::-1], 2)
        else:
            lexeme.cluster = 0
        lex_added += 1
    nlp.vocab.cfg.update({'oov_prob': oov_prob})

    if len(vectors_data):
        nlp.vocab.vectors = Vectors(data=vectors_data, keys=vector_keys)
    if prune_vectors >= 1:
        nlp.vocab.prune_vectors(prune_vectors)
    vec_added = len(nlp.vocab.vectors)

    prints("{} entries, {} vectors".format(lex_added, vec_added),
           title="Sucessfully compiled vocab")
    return nlp


def read_vectors(vectors_loc):
    print("Reading vectors...")
    with vectors_loc.open() as f:
        shape = tuple(int(size) for size in f.readline().split())
        vectors_data = numpy.zeros(shape=shape, dtype='f')
        vectors_keys = []
        for i, line in enumerate(tqdm(f)):
            pieces = line.split()
            if pieces:
                word = pieces.pop(0)
                try:
                    vectors_data[i] = numpy.array(
                        [float(val_str) for val_str in pieces],
                        dtype='f')
                    vectors_keys.append(word)
                except Exception as e:
                    logging.warning(str(e))
            else:
                logging.warning('Could not parse line: %s: %s', i, line)
    return vectors_data, vectors_keys


def read_binary_vectors(vectors_loc):
    print("Reading binary vectors...")

    from gensim.models import Word2Vec
    model = Word2Vec.load(str(vectors_loc))

    vectors_data = numpy.zeros(shape=model.wv.syn0.shape, dtype='f')
    vectors_keys = []

    for i, (word, vocab) in enumerate(
            sorted(model.wv.vocab.items(), key=lambda item: -item[1].count)):
        vectors_data[i] = model.wv.syn0[vocab.index]
        vectors_keys.append(word)

    return vectors_data, vectors_keys


def read_freqs(freqs_loc, max_length=100, min_doc_freq=5, min_freq=50):
    print("Counting frequencies...")
    counts = PreshCounter()
    total = 0
    with freqs_loc.open() as f:
        for i, line in enumerate(f):
            freq, doc_freq, key = line.rstrip().split('\t', 2)
            freq = int(freq)
            counts.inc(i + 1, freq)
            total += freq
    counts.smooth()
    log_total = math.log(total)
    probs = {}
    with freqs_loc.open() as f:
        for line in tqdm(f):
            freq, doc_freq, key = line.rstrip().split('\t', 2)
            doc_freq = int(doc_freq)
            freq = int(freq)
            if doc_freq >= min_doc_freq and freq >= min_freq and len(key) < max_length:
                smooth_count = counts.smoother(int(freq))
                probs[key] = math.log(smooth_count) - log_total
    oov_prob = math.log(counts.smoother(0)) - log_total
    return probs, oov_prob


def read_clusters(clusters_loc):
    print("Reading clusters...")
    clusters = {}
    with clusters_loc.open() as f:
        for line in tqdm(f):
            try:
                cluster, word, freq = line.split()
                word = fix_text(word)
            except ValueError:
                continue
            # If the clusterer has only seen the word a few times, its
            # cluster is unreliable.
            if int(freq) >= 3:
                clusters[word] = cluster
            else:
                clusters[word] = '0'
    # Expand clusters with re-casing
    for word, cluster in list(clusters.items()):
        if word.lower() not in clusters:
            clusters[word.lower()] = cluster
        if word.title() not in clusters:
            clusters[word.title()] = cluster
        if word.upper() not in clusters:
            clusters[word.upper()] = cluster
    return clusters