针对的问题:
所提方法的思想:
模型:
x是第一层输入层的激活向量,来自矩阵C的输入词特征的连接:
代码:来自https://github.com/graykode/nlp-tutorial/tree/master/1-1.NNLM
# code by Tae Hwan Jung @graykode import numpy as np import torch import torch.nn as nn import torch.optim as optim from torch.autograd import Variable dtype = torch.FloatTensor sentences = [ "i like dog", "i love coffee", "i hate milk"] word_list = " ".join(sentences).split() #制作词汇表 print(word_list) word_list = list(set(word_list)) #去重 print("after set: ",word_list) word_dict = {w: i for i, w in enumerate(word_list)} #每个单词对应的索引 number_dict = {i: w for i, w in enumerate(word_list)} #每个索引对应的单词 n_class = len(word_dict) # 单词总数 # NNLM Parameter n_step = 2 # 根据前两个单词预测第3个单词 n_hidden = 2 # h 隐藏层神经元的个数 m = 2 # m 词向量的维度 # 由于pytorch中输入的数据是以batch小批量进行输入的,下面的函数就是将原始数据以一个batch为基本单位喂给模型 def make_batch(sentences): input_batch = [] target_batch = [] for sen in sentences: word = sen.split() input = [word_dict[n] for n in word[:-1]] target = word_dict[word[-1]] input_batch.append(input) target_batch.append(target) return input_batch, target_batch # Model class NNLM(nn.Module): def __init__(self): super(NNLM, self).__init__() self.C = nn.Embedding(n_class, m) self.H = nn.Parameter(torch.randn(n_step * m, n_hidden).type(dtype)) self.W = nn.Parameter(torch.randn(n_step * m, n_class).type(dtype)) self.d = nn.Parameter(torch.randn(n_hidden).type(dtype)) self.U = nn.Parameter(torch.randn(n_hidden, n_class).type(dtype)) self.b = nn.Parameter(torch.randn(n_class).type(dtype)) def forward(self, X): X = self.C(X) X = X.view(-1, n_step * m) # [batch_size, n_step * n_class] tanh = torch.tanh(self.d + torch.mm(X, self.H)) # [batch_size, n_hidden] output = self.b + torch.mm(X, self.W) + torch.mm(tanh, self.U) # [batch_size, n_class] return output model = NNLM() criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=0.001) input_batch, target_batch = make_batch(sentences) input_batch = Variable(torch.LongTensor(input_batch)) target_batch = Variable(torch.LongTensor(target_batch)) # Training for epoch in range(5000): optimizer.zero_grad() output = model(input_batch) # output : [batch_size, n_class], target_batch : [batch_size] (LongTensor, not one-hot) loss = criterion(output, target_batch) if (epoch + 1)%1000 == 0: print(‘Epoch:‘, ‘%04d‘ % (epoch + 1), ‘cost =‘, ‘{:.6f}‘.format(loss)) loss.backward() optimizer.step() # Predict predict = model(input_batch).data.max(1, keepdim=True)[1] # Test print([sen.split()[:2] for sen in sentences], ‘->‘, [number_dict[n.item()] for n in predict.squeeze()])
结果:
A Neural Probabilistic Language Model_论文阅读及代码复现pytorch版
原文:https://www.cnblogs.com/harbin-ho/p/12801999.html