Hilary Clinton Mails Deciphered

from wordcloud import WordCloud # pip install wordcloud

# set max words to 200 to simplify visualization
# .txt file generated from original .csv data
corpus = open('data/extractedMail.txt').read()
wordcloud = WordCloud(height=1000, width=2000, max_words=200).generate(corpus)

plt.figure(figsize=(20, 10)) # plot
plt.imshow(wordcloud)
plt.axis("off")
plt.show()

from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.stem.snowball import EnglishStemmer

def preprocessing(corpus, stopwords, verify=True):
    # tokenize the corpus
    # nltk.download('punkt')
    token = nltk.word_tokenize(corpus)
    
    # remove the stopwords
    clean_token = [word for word in token if word not in stopwords]
    
    # lemmatization
    # nltk.download('wordnet')
    lemmatizer = WordNetLemmatizer()
    lemmatized_token = [lemmatizer.lemmatize(word) for word in clean_token]
    
    # stemming
    stemmer = EnglishStemmer()
    stemmed_token = [stemmer.stem(word) for word in lemmatized_token]
    
    # remove all the punctuations
    no_punctuation_token = [word for word in stemmed_token if word not in string.punctuation]
    
    # verify a bit the progress we made
    if verify:
        print('raw token size: ', len(token))
        print('token size after preprocessing: ', len(no_punctuation_token))
    
    return no_punctuation_token

from nltk.sentiment.vader import SentimentIntensityAnalyzer

# use compound value for polarity aggregation
extractedMail['Sentiment'] = np.nan

sid = SentimentIntensityAnalyzer()

def calc_sentiment(row):
    # nltk.download('vader_lexicon')
    if not extractedMail.loc[row, 'Countries']:
        extractedMail.loc[row, 'Sentiment'] = 0.0
    else:
        vader_points = sid.polarity_scores(' '.join(extractedMail.loc[row, 'Tokens']))
        extractedMail.loc[row, 'Sentiment'] = vader_points['compound']

for row in range(extractedMail.shape[0]):
    calc_sentiment(row)

from gensim import corpora, models, similarities

# make and save dictionary
dictionary = corpora.Dictionary(iter_corpus)
dictionary.save('dictionary.dict')

# make and save corpus
final_corpus = [dictionary.doc2bow(text) for text in iter_corpus]
corpora.MmCorpus.serialize('corpus.mm', final_corpus)


# load
dictionary = corpora.Dictionary.load('dictionary.dict')
corpus = corpora.MmCorpus('corpus.mm')

# run
model = models.LdaModel(corpus, id2word=dictionary, num_topics=10)
model.print_topics()