#!/usr/bin/env python
# encoding: utf-8
"""Produce a translation matrix to translate words from one language to another, using either
a standard nearest neighbour method or a globally corrected neighbour retrieval method [1]_.
This method can be used to augment the existing phrase tables with more candidate translations, or
filter out errors from the translation tables and known dictionaries [2]_. What's more, it also works
for any two sets of named-vectors where there are some paired-guideposts to learn the transformation.
Examples
--------
How to make translation between two set of word-vectors
=======================================================
Initialize two word-vector models
.. sourcecode:: pycon
>>> from gensim.models import KeyedVectors
>>> from gensim.test.utils import datapath
>>>
>>> model_en = KeyedVectors.load_word2vec_format(datapath("EN.1-10.cbow1_wind5_hs0_neg10_size300_smpl1e-05.txt"))
>>> model_it = KeyedVectors.load_word2vec_format(datapath("IT.1-10.cbow1_wind5_hs0_neg10_size300_smpl1e-05.txt"))
Define word pairs (that will be used for construction of translation matrix)
.. sourcecode:: pycon
>>> word_pairs = [
... ("one", "uno"), ("two", "due"), ("three", "tre"), ("four", "quattro"), ("five", "cinque"),
... ("seven", "sette"), ("eight", "otto"),
... ("dog", "cane"), ("pig", "maiale"), ("fish", "cavallo"), ("birds", "uccelli"),
... ("apple", "mela"), ("orange", "arancione"), ("grape", "acino"), ("banana", "banana")
... ]
Fit :class:`~gensim.models.translation_matrix.TranslationMatrix`
.. sourcecode:: pycon
>>> trans_model = TranslationMatrix(model_en, model_it, word_pairs=word_pairs)
Apply model (translate words "dog" and "one")
.. sourcecode:: pycon
>>> trans_model.translate(["dog", "one"], topn=3)
OrderedDict([('dog', [u'cane', u'gatto', u'cavallo']), ('one', [u'uno', u'due', u'tre'])])
Save / load model
.. sourcecode:: pycon
>>> with temporary_file("model_file") as fname:
... trans_model.save(fname) # save model to file
... loaded_trans_model = TranslationMatrix.load(fname) # load model
How to make translation between two :class:`~gensim.models.doc2vec.Doc2Vec` models
==================================================================================
Prepare data and models
.. sourcecode:: pycon
>>> from gensim.test.utils import datapath
>>> from gensim.test.test_translation_matrix import read_sentiment_docs
>>> from gensim.models import Doc2Vec
>>>
>>> data = read_sentiment_docs(datapath("alldata-id-10.txt"))[:5]
>>> src_model = Doc2Vec.load(datapath("small_tag_doc_5_iter50"))
>>> dst_model = Doc2Vec.load(datapath("large_tag_doc_10_iter50"))
Train backward translation
.. sourcecode:: pycon
>>> model_trans = BackMappingTranslationMatrix(data, src_model, dst_model)
>>> trans_matrix = model_trans.train(data)
Apply model
.. sourcecode:: pycon
>>> result = model_trans.infer_vector(dst_model.dv[data[3].tags])
References
----------
.. [1] Dinu, Georgiana, Angeliki Lazaridou, and Marco Baroni. "Improving zero-shot learning by mitigating the
hubness problem", https://arxiv.org/abs/1412.6568
.. [2] Tomas Mikolov, Ilya Sutskever, Kai Chen, Greg Corrado, and Jeffrey Dean.
"Distributed Representations of Words and Phrases and their Compositionality", https://arxiv.org/abs/1310.4546
"""
import warnings
from collections import OrderedDict
import numpy as np
from gensim import utils
class Space:
"""An auxiliary class for storing the the words space."""
def __init__(self, matrix, index2word):
"""
Parameters
----------
matrix : iterable of numpy.ndarray
Matrix that contains word-vectors.
index2word : list of str
Words which correspond to the `matrix`.
"""
self.mat = matrix
self.index2word = index2word
# build a dict to map word to index
self.word2index = {}
for idx, word in enumerate(self.index2word):
self.word2index[word] = idx
@classmethod
def build(cls, lang_vec, lexicon=None):
"""Construct a space class for the lexicon, if it's provided.
Parameters
----------
lang_vec : :class:`~gensim.models.keyedvectors.KeyedVectors`
Model from which the vectors will be extracted.
lexicon : list of str, optional
Words which contains in the `lang_vec`, if `lexicon = None`, the lexicon is all the lang_vec's word.
Returns
-------
:class:`~gensim.models.translation_matrix.Space`
Object that stored word-vectors
"""
# `words` to store all the words
# `mat` to store the word vector for each word in the 'words' list
words = []
mat = []
if lexicon is not None:
# if the lexicon is not provided, using all the Keyedvectors's words as default
for item in lexicon:
words.append(item)
mat.append(lang_vec.vectors[lang_vec.get_index(item)])
else:
for item in lang_vec.index_to_key:
words.append(item)
mat.append(lang_vec.vectors[lang_vec.get_index(item)])
return Space(mat, words)
def normalize(self):
"""Normalize the word vectors matrix."""
self.mat = self.mat / np.sqrt(np.sum(np.square(self.mat), axis=1, keepdims=True))
class TranslationMatrix(utils.SaveLoad):
"""Objects of this class realize the translation matrix which maps the source language to the target language.
The main methods are:
We map it to the other language space by computing z = Wx, then return the
word whose representation is close to z.
For details on use, see the tutorial notebook [3]_
Examples
--------
.. sourcecode:: pycon
>>> from gensim.models import KeyedVectors
>>> from gensim.test.utils import datapath
>>> en = datapath("EN.1-10.cbow1_wind5_hs0_neg10_size300_smpl1e-05.txt")
>>> it = datapath("IT.1-10.cbow1_wind5_hs0_neg10_size300_smpl1e-05.txt")
>>> model_en = KeyedVectors.load_word2vec_format(en)
>>> model_it = KeyedVectors.load_word2vec_format(it)
>>>
>>> word_pairs = [
... ("one", "uno"), ("two", "due"), ("three", "tre"), ("four", "quattro"), ("five", "cinque"),
... ("seven", "sette"), ("eight", "otto"),
... ("dog", "cane"), ("pig", "maiale"), ("fish", "cavallo"), ("birds", "uccelli"),
... ("apple", "mela"), ("orange", "arancione"), ("grape", "acino"), ("banana", "banana")
... ]
>>>
>>> trans_model = TranslationMatrix(model_en, model_it)
>>> trans_model.train(word_pairs)
>>> trans_model.translate(["dog", "one"], topn=3)
OrderedDict([('dog', [u'cane', u'gatto', u'cavallo']), ('one', [u'uno', u'due', u'tre'])])
References
----------
.. [3] https://github.com/RaRe-Technologies/gensim/blob/3.2.0/docs/notebooks/translation_matrix.ipynb
"""
def __init__(self, source_lang_vec, target_lang_vec, word_pairs=None, random_state=None):
"""
Parameters
----------
source_lang_vec : :class:`~gensim.models.keyedvectors.KeyedVectors`
Word vectors for source language.
target_lang_vec : :class:`~gensim.models.keyedvectors.KeyedVectors`
Word vectors for target language.
word_pairs : list of (str, str), optional
Pairs of words that will be used for training.
random_state : {None, int, array_like}, optional
Seed for random state.
"""
self.source_word = None
self.target_word = None
self.source_lang_vec = source_lang_vec
self.target_lang_vec = target_lang_vec
self.random_state = utils.get_random_state(random_state)
self.translation_matrix = None
self.source_space = None
self.target_space = None
if word_pairs is not None:
if len(word_pairs[0]) != 2:
raise ValueError("Each training data item must contain two different language words.")
self.train(word_pairs)
def train(self, word_pairs):
"""Build the translation matrix to map from source space to target space.
Parameters
----------
word_pairs : list of (str, str), optional
Pairs of words that will be used for training.
"""
self.source_word, self.target_word = zip(*word_pairs)
self.source_space = Space.build(self.source_lang_vec, set(self.source_word))
self.target_space = Space.build(self.target_lang_vec, set(self.target_word))
self.source_space.normalize()
self.target_space.normalize()
m1 = self.source_space.mat[[self.source_space.word2index[item] for item in self.source_word], :]
m2 = self.target_space.mat[[self.target_space.word2index[item] for item in self.target_word], :]
self.translation_matrix = np.linalg.lstsq(m1, m2, -1)[0]
def save(self, *args, **kwargs):
"""Save the model to a file. Ignores (doesn't store) the `source_space` and `target_space` attributes."""
kwargs['ignore'] = kwargs.get('ignore', ['source_space', 'target_space'])
super(TranslationMatrix, self).save(*args, **kwargs)
def apply_transmat(self, words_space):
"""Map the source word vector to the target word vector using translation matrix.
Parameters
----------
words_space : :class:`~gensim.models.translation_matrix.Space`
`Space` object constructed for the words to be translated.
Returns
-------
:class:`~gensim.models.translation_matrix.Space`
`Space` object constructed for the mapped words.
"""
return Space(np.dot(words_space.mat, self.translation_matrix), words_space.index2word)
def translate(self, source_words, topn=5, gc=0, sample_num=None, source_lang_vec=None, target_lang_vec=None):
"""Translate the word from the source language to the target language.
Parameters
----------
source_words : {str, list of str}
Single word or a list of words to be translated
topn : int, optional
Number of words that will be returned as translation for each `source_words`
gc : int, optional
Define translation algorithm, if `gc == 0` - use standard NN retrieval,
otherwise, use globally corrected neighbour retrieval method (as described in [1]_).
sample_num : int, optional
Number of words to sample from the source lexicon, if `gc == 1`, then `sample_num` **must** be provided.
source_lang_vec : :class:`~gensim.models.keyedvectors.KeyedVectors`, optional
New source language vectors for translation, by default, used the model's source language vector.
target_lang_vec : :class:`~gensim.models.keyedvectors.KeyedVectors`, optional
New target language vectors for translation, by default, used the model's target language vector.
Returns
-------
:class:`collections.OrderedDict`
Ordered dict where each item is `word`: [`translated_word_1`, `translated_word_2`, ...]
"""
if isinstance(source_words, str):
# pass only one word to translate
source_words = [source_words]
# If the language word vector not provided by user, use the model's
# language word vector as default
if source_lang_vec is None:
warnings.warn(
"The parameter source_lang_vec isn't specified, "
"use the model's source language word vector as default."
)
source_lang_vec = self.source_lang_vec
if target_lang_vec is None:
warnings.warn(
"The parameter target_lang_vec isn't specified, "
"use the model's target language word vector as default."
)
target_lang_vec = self.target_lang_vec
# If additional is provided, bootstrapping vocabulary from the source language word vector model.
if gc:
if sample_num is None:
raise RuntimeError(
"When using the globally corrected neighbour retrieval method, "
"the `sample_num` parameter(i.e. the number of words sampled from source space) must be provided."
)
lexicon = set(source_lang_vec.index_to_key)
addition = min(sample_num, len(lexicon) - len(source_words))
lexicon = self.random_state.choice(list(lexicon.difference(source_words)), addition)
source_space = Space.build(source_lang_vec, set(source_words).union(set(lexicon)))
else:
source_space = Space.build(source_lang_vec, source_words)
target_space = Space.build(target_lang_vec, )
# Normalize the source vector and target vector
source_space.normalize()
target_space.normalize()
# Map the source language to the target language
mapped_source_space = self.apply_transmat(source_space)
# Use the cosine similarity metric
sim_matrix = -np.dot(target_space.mat, mapped_source_space.mat.T)
# If `gc=1`, using corrected retrieval method
if gc:
srtd_idx = np.argsort(np.argsort(sim_matrix, axis=1), axis=1)
sim_matrix_idx = np.argsort(srtd_idx + sim_matrix, axis=0)
else:
sim_matrix_idx = np.argsort(sim_matrix, axis=0)
# Translate the words and for each word return the `topn` similar words
translated_word = OrderedDict()
for idx, word in enumerate(source_words):
translated_target_word = []
# Search the most `topn` similar words
for j in range(topn):
map_space_id = sim_matrix_idx[j, source_space.word2index[word]]
translated_target_word.append(target_space.index2word[map_space_id])
translated_word[word] = translated_target_word
return translated_word
class BackMappingTranslationMatrix(utils.SaveLoad):
"""Realize the BackMapping translation matrix which maps the source model's document vector
to the target model's document vector (old model).
BackMapping translation matrix is used to learn a mapping for two document vector spaces which we
specify as source document vector and target document vector. The target document vectors are trained
on a superset corpus of source document vectors; we can incrementally increase the vector in
the old model through the BackMapping translation matrix.
For details on use, see the tutorial notebook [3]_.
Examples
--------
.. sourcecode:: pycon
>>> from gensim.test.utils import datapath
>>> from gensim.test.test_translation_matrix import read_sentiment_docs
>>> from gensim.models import Doc2Vec, BackMappingTranslationMatrix
>>>
>>> data = read_sentiment_docs(datapath("alldata-id-10.txt"))[:5]
>>> src_model = Doc2Vec.load(datapath("small_tag_doc_5_iter50"))
>>> dst_model = Doc2Vec.load(datapath("large_tag_doc_10_iter50"))
>>>
>>> model_trans = BackMappingTranslationMatrix(src_model, dst_model)
>>> trans_matrix = model_trans.train(data)
>>>
>>> result = model_trans.infer_vector(dst_model.dv[data[3].tags])
"""
def __init__(self, source_lang_vec, target_lang_vec, tagged_docs=None, random_state=None):
"""
Parameters
----------
source_lang_vec : :class:`~gensim.models.doc2vec.Doc2Vec`
Source Doc2Vec model.
target_lang_vec : :class:`~gensim.models.doc2vec.Doc2Vec`
Target Doc2Vec model.
tagged_docs : list of :class:`~gensim.models.doc2vec.TaggedDocument`, optional.
Documents that will be used for training, both the source language document vector and
target language document vector trained on those tagged documents.
random_state : {None, int, array_like}, optional
Seed for random state.
"""
self.tagged_docs = tagged_docs
self.source_lang_vec = source_lang_vec
self.target_lang_vec = target_lang_vec
self.random_state = utils.get_random_state(random_state)
self.translation_matrix = None
if tagged_docs is not None:
self.train(tagged_docs)
def train(self, tagged_docs):
"""Build the translation matrix to map from the source model's vectors to target model's vectors
Parameters
----------
tagged_docs : list of :class:`~gensim.models.doc2vec.TaggedDocument`, Documents
that will be used for training, both the source language document vector and
target language document vector trained on those tagged documents.
Returns
-------
numpy.ndarray
Translation matrix that maps from the source model's vectors to target model's vectors.
"""
m1 = [self.source_lang_vec.dv[item.tags].flatten() for item in tagged_docs]
m2 = [self.target_lang_vec.dv[item.tags].flatten() for item in tagged_docs]
self.translation_matrix = np.linalg.lstsq(m2, m1, -1)[0]
return self.translation_matrix
def infer_vector(self, target_doc_vec):
"""Translate the target model's document vector to the source model's document vector
Parameters
----------
target_doc_vec : numpy.ndarray
Document vector from the target document, whose document are not in the source model.
Returns
-------
numpy.ndarray
Vector `target_doc_vec` in the source model.
"""
return np.dot(target_doc_vec, self.translation_matrix)