分析《 Kinopoisk》评论的情感色彩

参赛作品

数据语料库的形成

import requests from bs4 import BeautifulSoup import numpy as np import time import os def load_data(url): r = requests.get(url, headers = headers) #  http  soup = BeautifulSoup(r.text, 'html.parser')#  html  reviews = soup.find_all(class_='_reachbanner_')#    reviews_clean = [] for review in reviews:#    html  reviews_clean.append(review.find_all(text=True)) return reviews_clean 

 def convert(reviews): #     review_converted = [] for review in reviews: for i in review: map(str, i) review = ''.join(review) review_converted.append(review) return review_converted def get_name(url): #    r = requests.get(url, headers = headers) soup = BeautifulSoup(r.text, 'html.parser') name = soup.find(class_='alternativeHeadline') name_clean = name.find_all(text = True) #   , . .     return str(name_clean[0]) 

 def parsing(url, status, path): page = 1 delays = [11, 12, 13, 11.5, 12.5, 13.5, 11.2, 12.3, 11.8] name = get_name(url) time.sleep(np.random.choice(delays)) #    while True: loaded_data = load_data(url + 'reviews/ord/date/status/{}/perpage/200/page/{}/'.format(status, page)) if loaded_data == []: break else: # E     ,    if not os.path.exists(path + r'\{}'.format(status)): os.makedirs(path + r'\{}'.format(status)) converted_data = convert(loaded_data) #   for i, review in enumerate(converted_data): with open(path + r'\{}\{}_{}_{}.txt'.format(status, name, page, i), 'w', encoding = 'utf-8') as output: output.write(review) page += 1 time.sleep(np.random.choice(delays)) 

 path = #    urles = #    statuses = ['good', 'bad', 'neutral'] delays = [15, 20, 13, 18, 12.5, 13.5, 25, 12.3, 23] for url in urles: for status in statuses: try: parsing(url = url, status = status, path=path) print('one category done') time.sleep(np.random.choice(delays)) #       AttributeError except AttributeError: print(' : {}, {}'.format(url, status)) break #  else  ,      #    ,     else: print('one url done') continue break 

预处理

 from nltk.corpus import PlaintextCorpusReader from nltk.stem.snowball import SnowballStemmer from nltk.probability import FreqDist from nltk.tokenize import RegexpTokenizer from nltk import bigrams from nltk import pos_tag from collections import OrderedDict from sklearn.metrics import classification_report, accuracy_score from sklearn.naive_bayes import MultinomialNB from sklearn.model_selection import GridSearchCV from sklearn.utils import shuffle from multiprocessing import Pool import numpy as np from scipy.sparse import csr_matrix 

 def lower_pos_tag(words): lower_words = [] for i in words: lower_words.append(i.lower()) pos_words = pos_tag(lower_words, lang='rus') return pos_words 

 def clean(words): stemmer = SnowballStemmer("russian") cleaned_words = [] for i in words: if i[1] in ['S', 'A', 'V', 'ADV']: cleaned_words.append(stemmer.stem(i[0])) return cleaned_words 

 corpus_root = #    def process(label): # Wordmatrix -     # All words -    data = {'Word_matrix': [], 'All_words': []} #      templist_allwords = [] #        corpus = PlaintextCorpusReader(corpus_root + '\\' + label, '.*', encoding='utf-8') #       names = corpus.fileids() #   tokenizer = RegexpTokenizer(r'\w+|[^\w\s]+') for i in range(len(names)): #   bag_words = tokenizer.tokenize(corpus.raw(names[i])) lower_words = lower_pos_tag(bag_words) cleaned_words = clean(lower_words) finalist = list(bigrams(cleaned_words)) + cleaned_words data['Word_matrix'].append(final_words) templist_allwords.extend(cleaned_words) #   templistfreq = FreqDist(templist_allwords) hapaxes = templistfreq.hapaxes() #    for word in templist_allwords: if word not in hapaxes: data['All_words'].append(word) return {label: data} 

 if __name__ == '__main__': data = {} labels = ['neutral', 'bad', 'good'] p = Pool(3) result = p.map(process, labels) for i in result: data.update(i) p.close() 

向量化

 #     : # [([  ], _)] labels = ['neutral', 'bad', 'good'] labeled_data = [] for label in labels: for document in data[label]['Word_matrix']: labeled_data.append((document, label)) #      all_words = [] for label in labels: frequency = FreqDist(data[label]['All_words'] common_words = frequency.most_common(10000) words = [i[0] for i in common_words] all_words.extend(words) #    unique_words = list(OrderedDict.fromkeys(all_words)) 

 #     nltk  : # # [({ : -   },  )] prepared_data = [] for x in labeled_data: d = defaultdict(int) for word in unique_words: if word in x[0]: d[word] += 1 if word not in x[0]: d[word] = 0 prepared_data.append((d, x[1])) #     scikit-learn #     matrix_vec = csr_matrix((len(labeled_data), len(unique_words)), dtype=np.int8).toarray() #     target = np.zeros(len(labeled_data), 'str') for index_doc, document in enumerate(labeled_data): for index_word, word in enumerate(unique_words): #  -     matrix_vec[index_doc, index_word] = document[0].count(word) target[index_doc] = document[1] #   X, Y = shuffle(matrix_vec, target) 

模型训练

 parameter = [1, 0, 0.1, 0.01, 0.001, 0.0001] param_grid = {'alpha': parameter} grid_search = GridSearchCV(MultinomialNB(), param_grid, cv=5) grid_search.fit(X, Y) Alpha, best_score = grid_search.best_params_, grid_search.best_score_ 

 model = MultinomialNB(0.1) model.fit(X, Y) # X_control, Y_control   ,   X  Y #        predicted = model.predict(X_control) #     score_test = accuracy_score(Y_control, predicted) #   report = classification_report(Y_control, predicted) 

结论