검색

import pandas as pd
df = pd.read_csv('neurips.zip')
df.head()

query = {'natural', 'language'}

import re
def tokenize(text):
    text = text.lower() # 소문자로 변환
    return re.findall(r'\w{2,}', text) # 2글자 이상 단어 추출

%%time
results = []
for row in df.itertuples():
    words = set(tokenize(row.abstract))
    if query < words: # 검색어가 부분집합이면
        results.append(row.Index)

results

df.loc[results]

a = list(range(1000000))

%%time
a.index(999999)

b = dict(zip(a, a))

%%time
b[999999]

from collections import defaultdict
index = defaultdict(set)

for row in df.itertuples():
    words = tokenize(row.abstract)
    for word in words:
        index[word].add(row.Index)

index['language']

%%time
results = list(index['natural'] & index['language'])

from collections import Counter
idxs = list(index['natural'] & index['language'])
results = []
for row in df.iloc[idxs].itertuples():
    words = tokenize(row.abstract)
    cnt = Counter(words)
    tf = sum(cnt[w] for w in query)
    results.append((tf, row.Index))

idx = [i for _, i in sorted(results, reverse=True)]

df.iloc[idx]

{k: len(v) for k, v in index.items()}

N, _ = df.shape

import numpy as np
idf = {k: np.log(N / len(v)) for k, v in index.items()}

idxs = list(index['natural'] & index['language'])
results = []

for row in df.iloc[idxs].itertuples():
    words = tokenize(row.abstract)
    cnt = Counter(words)
    tfidf = sum(cnt[w] * idf[w] for w in query)
    results.append((tfidf, row.Index))

idx = [i for _, i in sorted(results, reverse=True)]

!pip install rank_bm25 kiwipiepy

import pandas as pd
books = pd.read_csv('science_books.csv')

from kiwipiepy import Kiwi
kiwi = Kiwi()


def tokenize(sent):
    for token in kiwi.tokenize(sent):
        if token.tag in {'NNG', 'NNP', 'SL', 'VV', 'VA'}:
            yield token.form, token.tag

tokenized_corpus = []
for title in books.제목:
    tokenized_corpus.append(list(tokenize(title)))

from rank_bm25 import BM25Okapi
bm25 = BM25Okapi(tokenized_corpus)

import pandas as pd
idf_table = pd.DataFrame(bm25.idf.items(), columns=['token', 'idf'])
idf_table.sort_values('idf')

query = list(tokenize('다정한 것이 살아남는다'))
bm25.get_top_n(query, books.제목, n=5)

!pip install sentence-transformers

from sentence_transformers import SentenceTransformer
sbert = SentenceTransformer(
    'snunlp/KR-SBERT-V40K-klueNLI-augSTS')

from sklearn.metrics.pairwise import cosine_similarity
emb = sbert.encode(books.제목)

query_emb = sbert.encode(['다정한 것이 살아남는다'])

from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

%%time
sims = cosine_similarity(query_emb, emb)
ids = np.argsort(-sims[0])[:5]

books.iloc[ids]

pip install nmslib

import nmslib
index = nmslib.init()
index.addDataPointBatch(emb)
index.createIndex()

%%time
ids, dist = index.knnQuery(query_emb, k=5)

books.iloc[ids]

!pip install chromadb

import chromadb

client = chromadb.Client()

collection = client.create_collection(
    name="science_books", 
    embedding_function=sbert.encode)

metadatas = books.제목.map(lambda x: {'length': len(x)}).tolist()
ids = books.index.map(str).tolist()
collection.add(
    documents=books.제목.tolist(),
    metadatas=metadatas,
    ids=ids
)

results = collection.query(
    query_texts=["다정한 것이 살아남는다"],
    n_results=5
)

%%time
results = collection.query(
    query_embeddings=[query_emb[0]],
    n_results=5
)

query_emb.shape

results

results = collection.query(
    query_texts=["다정한 것이 살아남는다"],
    n_results=5,
    where={"length": {'$lt': 75}},
    where_document={"$contains":"다정"}
)

results