由浅入深弄懂simhash来比较文本的相似度

# -*- coding: utf-8 -*-

import jieba.analyse
import jieba
import json
import numpy as np

class SimHash(object):

    # 文本使用simhash方法，转成64维向量
    def content_vector(self, contents):
        # 获取关键词及其tf-idf权重值
        # 分词 hash 加权
        keywords = jieba.analyse.extract_tags(contents, withWeight=True)
        ret_list = []
        for word, weight in keywords:
            # hash
            word_hash = bin(hash(word)).replace(‘0b‘, ‘‘).replace(‘-‘, ‘‘).zfill(64)
            weight = int(weight)
            tmp_list = []
            for feature in word_hash:
                if feature == ‘1‘:
                    # 加权
                    tmp_list.append(weight)
                else:
                    tmp_list.append(-1 * weight)
            ret_list.append(tmp_list)
        
        # 降维
        sum_list = np.sum(np.array(ret_list), axis=0)
        res_str = ‘‘
        for i in sum_list:
            if i > 0:
                res_str += ‘1‘
            else:
                res_str += ‘0‘
        return res_str

    # 计算两个向量的海明距离
    def cal_hamming_distance(self, vector1, vector2):
        vec1_int = int((‘0b‘+ vector1), 2)
        vec2_int = int((‘0b‘ +vector2), 2)
        # 异或操作
        num = vec1_int ^ vec2_int
        # 获取num中1的个数，即为海明距离
        count = 0
        for i in bin(num).replace(‘0b‘, ‘‘):
            if i == ‘1‘:
                count += 1
        return count