감성분석

import pandas as pd
df = pd.read_excel('yelp.xlsx')
df.head()

from sklearn.feature_extraction.text import CountVectorizer
cv = CountVectorizer(max_features=1000, stop_words='english')
dtm = cv.fit_transform(df.review)

x = dtm
y = df.sentiment

from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(
    x,
    y,
    test_size=0.2,   # 20%의 데이터를 테스트용으로 유보
    random_state=42) # 유사난수의 씨앗값 seed을 42로 설정

from sklearn.naive_bayes import BernoulliNB

model = BernoulliNB()

model.fit(x_train, y_train)

model.score(x_train, y_train)

model.score(x_test, y_test)

prob_df = pd.DataFrame({
    '단어': cv.get_feature_names_out(),
    '비율': model.feature_log_prob_[1] - model.feature_log_prob_[0]
})

prob_df.sort_values('비율').tail(10)

prob_df.sort_values('비율').head(10)

from sklearn.linear_model import LogisticRegressionCV

model = LogisticRegressionCV(
    penalty='elasticnet', solver='saga', random_state=42,
    Cs=[0.1, 1, 10], l1_ratios=[0, 0.5, 1], max_iter=4000)

model.fit(x_train, y_train)

model.C_

model.l1_ratio_

model.score(x_train, y_train)

model.score(x_test, y_test)

word_coef = pd.DataFrame({
    '단어': cv.get_feature_names_out(),
    '가중치': model.coef_.flat
})

word_coef.sort_values('가중치').tail(10)

word_coef.sort_values('가중치').head(10)

y_pred = model.predict(x_test)

probs = model.predict_proba(x_test)

prob = probs[:, 1]

import numpy as np
threshold = 0.5 # 문턱값
y_pred = np.where(prob > threshold, 1, 0)