Cacher is the code snippet organizer for pro developers

library(dplyr)

tokens <- read.csv('data/tokens.csv', as.is = TRUE)


result <- tokens %>%
    group_by(native, id) %>%
    summarise(
        n_token = n(),
        n_type  = n_distinct(token),
        n_sent  = n_distinct(sen_no),

        ttr     = n_type / n_token,
        gi      = n_type / sqrt(n_token),

        awl     = mean(nchar(token)),
        asl     = n_token / n_sent
    )


write.csv(result, file = 'out/2015.12.04-awl_asl.csv', row.names = FALSE)

result %>%
    mutate_each(
        funs(round(., 2)),
        ttr:asl
    ) %>%
    as.data.frame() %>%
    print()