使用Python和TextBlob进行原型情感分析

选择组件

import collections import nltk import os from sklearn import ( datasets, model_selection, feature_extraction, linear_model ) def extract_features(corpus): '''Extract TF-IDF features from corpus''' # vectorize means we turn non-numerical data into an array of numbers count_vectorizer = feature_extraction.text.CountVectorizer( lowercase=True, # for demonstration, True by default tokenizer=nltk.word_tokenize, # use the NLTK tokenizer stop_words='english', # remove stop words min_df=1 # minimum document frequency, ie the word must appear more than once. ) processed_corpus = count_vectorizer.fit_transform(corpus) processed_corpus = feature_extraction.text.TfidfTransformer().fit_transform( processed_corpus) return processed_corpus data_directory = 'movie_reviews' movie_sentiment_data = datasets.load_files(data_directory, shuffle=True) print('{} files loaded.'.format(len(movie_sentiment_data.data))) print('They contain the following classes: {}.'.format( movie_sentiment_data.target_names)) movie_tfidf = extract_features(movie_sentiment_data.data) X_train, X_test, y_train, y_test = model_selection.train_test_split( movie_tfidf, movie_sentiment_data.target, test_size=0.30, random_state=42) # similar to nltk.NaiveBayesClassifier.train() model = linear_model.LogisticRegression() model.fit(X_train, y_train) print('Model performance: {}'.format(model.score(X_test, y_test))) y_pred = model.predict(X_test) for i in range(5): print('Review:\n{review}\n-\nCorrect label: {correct}; Predicted: {predict}'.format( review=X_test[i], correct=y_test[i], predict=y_pred[i] )) 

1. 下载数据以进行模型训练（第25-26行）
2. 准备上传的数据-特征提取（第31-34行）
3. 创建和训练模型（第36-39行）
4. 测试训练好的模型并输出结果（第41-45行）

准备资料

下载资料

 import pandas as pd data = pd.read_csv(data_path, delimiter) 

1. 与数据源的连接是针对CSV的，在我们的示例中，它是在read_csv函数内部实现的；
2. 支持格式功能；
3. 初步数据准备。

 import numpy as np # linear algebra import pandas as pd # data processing, CSV file I/O (eg pd.read_csv) import logging log = logging.getLogger() class CsvSentimentDataLoader(): def __init__(self, file_path, delim, text_attr, rate_attr, pos_rates): self.data_path = file_path self.delimiter = delim self.text_attr = text_attr self.rate_attr = rate_attr self.pos_rates = pos_rates def load_data(self): #     csv  tsv  data = pd.read_csv(self.data_path, self.delimiter) data.head() #   , #      data = data[[self.text_attr, self.rate_attr]] #    #     'pos'  'neg' data[self.rate_attr] = np.where( data[self.rate_attr].isin(self.pos_rates), 'pos', 'neg') return data 

我们将数据分为测试集和训练集

 from sklearn.model_selection import train_test_split # to split the training and testing data import logging log = logging.getLogger() class SimpleDataSplitter(): def __init__(self, text_attr, rate_attr, test_part_size=.3): self.text_attr = text_attr self.rate_attr = rate_attr self.test_part_size = test_part_size def split_data(self, data): x = data[self.text_attr] y = data[self.rate_attr] x_train, x_test, y_train, y_test = train_test_split( x, y, test_size = self.test_part_size) return x_train, x_test, y_train, y_test 

数据集

• Amazon Alexa评论数据集， 可在Kaggle获得
• 加州大学情感标记句子数据集

 import os import collections import logging from web.data.loaders import CsvSentimentDataLoader from web.data.splitters import SimpleDataSplitter, TdIdfDataSplitter log = logging.getLogger() class AmazonAlexaDataset(): def __init__(self): self.file_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'amazon_alexa/train.tsv')) self.delim = '\t' self.text_attr = 'verified_reviews' self.rate_attr = 'feedback' self.pos_rates = [1] self.data = None self.train = None self.test = None def load_data(self): loader = CsvSentimentDataLoader(self.file_path, self.delim, self.text_attr, self.rate_attr, self.pos_rates) splitter = SimpleDataSplitter(self.text_attr, self.rate_attr, test_part_size=.3) self.data = loader.load_data() x_train, x_test, y_train, y_test = splitter.split_data(self.data) self.train = [x for x in zip(x_train, y_train)] self.test = [x for x in zip(x_test, y_test)] 

我们训练模型

 class TextBlobWrapper(): def __init__(self): self.log = logging.getLogger() self.is_model_trained = False self.classifier = None def init_app(self): self.log.info('>>>>> TextBlob initialization started') self.ensure_model_is_trained() self.log.info('>>>>> TextBlob initialization completed') def ensure_model_is_trained(self): if not self.is_model_trained: ds = SentimentLabelledDataset() ds.load_data() # train the classifier and test the accuracy self.classifier = NaiveBayesClassifier(ds.train) acr = self.classifier.accuracy(ds.test) self.log.info(str.format('>>>>> NaiveBayesClassifier trained with accuracy {}', acr)) self.is_model_trained = True return self.classifier 

测试中

  @loggingtestcase.capturelogs(None, level='INFO') def test_classifier_on_separate_set(self, logs): tb = TextBlobWrapper() # Going to be trained on Amazon Alexa dataset ds = SentimentLabelledDataset() # Test dataset ds.load_data() # Check poisitives true_pos = 0 data = ds.data.to_numpy() seach_mask = np.isin(data[:, 1], ['pos']) data = data[seach_mask][:100] for e in data[:]: # Model train will be performed on first classification call r = tb.do_sentiment_classification(e[0]) if r == e[1]: true_pos += 1 self.assertLessEqual(true_pos, 100) print(str.format('\n\nTrue Positive answers - {} of 100', true_pos)) 

• 下载测试数据；
• 接受100个标记为“ pos”的帖子
• 我们通过模型运行它们中的每一个并计算正确结果的数量
• 在控制台中显示最终结果。

结果

更改代码并测试，运行

总结