t_wの輪郭

何も分かってないけどajinkyaT/albert-japanese-v2-finetuned-nerでsentence embeddingっぽいものが取れて、大体良い感じの類似度が出る。

from transformers import (
    TFAutoModelForMaskedLM, AutoTokenizer, TFAutoModel
)
import tensorflow as tf
import tensorflowjs as tfjs
from tensorflow import keras
import numpy as np

tokenizer = AutoTokenizer.from_pretrained("ajinkyaT/albert-japanese-v2-finetuned-ner")
model = TFAutoModel.from_pretrained("ajinkyaT/albert-japanese-v2-finetuned-ner")
print(model.summary())

def embedding_calc(text):
    tokens = tokenizer(text, return_tensors="tf")
    return model.layers[0].get_input_embeddings().call(tokens["input_ids"])[0]
    
def cos_sim(v1, v2):
    return np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))

embedding1 = tf.reduce_sum(embedding_calc("東京は日本の首都です"), 0)
embedding2 = tf.reduce_sum(embedding_calc("日本の首都は東京です"), 0)
embedding3 = tf.reduce_sum(embedding_calc("this is test sentence to test"), 0)
embedding4 = tf.reduce_sum(embedding_calc("japanese capital is Tokyo"), 0)

print(cos_sim(embedding1.numpy(), embedding2.numpy()))  # 0.9314501
print(cos_sim(embedding2.numpy(), embedding4.numpy()))  # 0.66796505
print(cos_sim(embedding2.numpy(), embedding3.numpy()))  # 0.44220626