#Things to note for getting the word frequency count is that we have to use the correct encoding.utf-8 in our case.
import re
import operator
import codecs
import collections
corpus=codecs.open('corpus', 'r', encoding='utf-8')
f=corpus.read()
corpus.close()
lines=f.split('\n')
wordlist=[]#create an empty wordlist
word_count=collections.Counter()
for line in lines:
	line=line.split('\t')[-1]
	line=re.sub('[०१२३४५६७८९]+',' ',line)#Remove hindi digits
	line=re.sub('[A-Za-z0-9]',' ',line)#Remove English digits +  Alphabets + numbers
	line=re.sub('[ .:;,\"\'।‘\(\)]+',' ',line) #Remove other identifiers used in grammar
	line=re.sub('[.]+',' ',line)#Remove identifiers like -,.,|(poorn viram in hindi).
	line=re.sub('[|-]+',' ',line)
	words=line.split(' ')
	for word in words:
		if re.search('[\n\t ]',word) or word=='' or word=='-':
			words.remove(word)
			continue
		word_count[word] = word_count[word]+ 1	
word_count_list=word_count.most_common(100000)
temp=sorted(word_count_list, key=lambda x: -x[1])
file_output=codecs.open('output.txt', 'w', encoding='utf-8')
for item in temp:
	file_output.write(str(item[1])+" "+item[0]+"\n")
file_output.close()
