#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2012 Radim Rehurek
# Copyright (C) 2017 Mohit Rathore
# Licensed under the GNU LGPL v2.1 - https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
"""This module implements functionality related to the `Term Frequency - Inverse Document Frequency
`_ class of bag-of-words vector space models.
"""
import logging
from functools import partial
import re
import numpy as np
from gensim import interfaces, matutils, utils
from gensim.utils import deprecated
logger = logging.getLogger(__name__)
def resolve_weights(smartirs):
"""Check the validity of `smartirs` parameters.
Parameters
----------
smartirs : str
`smartirs` or SMART (System for the Mechanical Analysis and Retrieval of Text)
Information Retrieval System, a mnemonic scheme for denoting tf-idf weighting
variants in the vector space model. The mnemonic for representing a combination
of weights takes the form ddd, where the letters represents the term weighting of the document vector.
for more information visit `SMART Information Retrieval System
`_.
Returns
-------
str of (local_letter, global_letter, normalization_letter)
local_letter : str
Term frequency weighing, one of:
* `b` - binary,
* `t` or `n` - raw,
* `a` - augmented,
* `l` - logarithm,
* `d` - double logarithm,
* `L` - log average.
global_letter : str
Document frequency weighting, one of:
* `x` or `n` - none,
* `f` - idf,
* `t` - zero-corrected idf,
* `p` - probabilistic idf.
normalization_letter : str
Document normalization, one of:
* `x` or `n` - none,
* `c` - cosine,
* `u` - pivoted unique,
* `b` - pivoted character length.
Raises
------
ValueError
If `smartirs` is not a string of length 3 or one of the decomposed value
doesn't fit the list of permissible values.
"""
if isinstance(smartirs, str) and re.match(r"...\....", smartirs):
match = re.match(r"(?P...)\.(?P...)", smartirs)
raise ValueError(
"The notation {ddd}.{qqq} specifies two term-weighting schemes, "
"one for collection documents ({ddd}) and one for queries ({qqq}). "
"You must train two separate tf-idf models.".format(
ddd=match.group("ddd"),
qqq=match.group("qqq"),
)
)
if not isinstance(smartirs, str) or len(smartirs) != 3:
raise ValueError("Expected a string of length 3 got " + smartirs)
w_tf, w_df, w_n = smartirs
if w_tf not in 'btnaldL':
raise ValueError("Expected term frequency weight to be one of 'btnaldL', got {}".format(w_tf))
if w_df not in 'xnftp':
raise ValueError("Expected inverse document frequency weight to be one of 'xnftp', got {}".format(w_df))
if w_n not in 'xncub':
raise ValueError("Expected normalization weight to be one of 'xncub', got {}".format(w_n))
# resolve aliases
if w_tf == "t":
w_tf = "n"
if w_df == "x":
w_df = "n"
if w_n == "x":
w_n = "n"
return w_tf + w_df + w_n
def df2idf(docfreq, totaldocs, log_base=2.0, add=0.0):
r"""Compute inverse-document-frequency for a term with the given document frequency `docfreq`:
:math:`idf = add + log_{log\_base} \frac{totaldocs}{docfreq}`
Parameters
----------
docfreq : {int, float}
Document frequency.
totaldocs : int
Total number of documents.
log_base : float, optional
Base of logarithm.
add : float, optional
Offset.
Returns
-------
float
Inverse document frequency.
"""
return add + np.log(float(totaldocs) / docfreq) / np.log(log_base)
def precompute_idfs(wglobal, dfs, total_docs):
"""Pre-compute the inverse document frequency mapping for all terms.
Parameters
----------
wglobal : function
Custom function for calculating the "global" weighting function.
See for example the SMART alternatives under :func:`~gensim.models.tfidfmodel.smartirs_wglobal`.
dfs : dict
Dictionary mapping `term_id` into how many documents did that term appear in.
total_docs : int
Total number of documents.
Returns
-------
dict of (int, float)
Inverse document frequencies in the format `{term_id_1: idfs_1, term_id_2: idfs_2, ...}`.
"""
# not strictly necessary and could be computed on the fly in TfidfModel__getitem__.
# this method is here just to speed things up a little.
return {termid: wglobal(df, total_docs) for termid, df in dfs.items()}
def smartirs_wlocal(tf, local_scheme):
"""Calculate local term weight for a term using the weighting scheme specified in `local_scheme`.
Parameters
----------
tf : int
Term frequency.
local : {'b', 'n', 'a', 'l', 'd', 'L'}
Local transformation scheme.
Returns
-------
float
Calculated local weight.
"""
if local_scheme == "n":
return tf
elif local_scheme == "l":
return 1 + np.log2(tf)
elif local_scheme == "d":
return 1 + np.log2(1 + np.log2(tf))
elif local_scheme == "a":
return 0.5 + (0.5 * tf / tf.max(axis=0))
elif local_scheme == "b":
return tf.astype('bool').astype('int')
elif local_scheme == "L":
return (1 + np.log2(tf)) / (1 + np.log2(tf.mean(axis=0)))
def smartirs_wglobal(docfreq, totaldocs, global_scheme):
"""Calculate global document weight based on the weighting scheme specified in `global_scheme`.
Parameters
----------
docfreq : int
Document frequency.
totaldocs : int
Total number of documents.
global_scheme : {'n', 'f', 't', 'p'}
Global transformation scheme.
Returns
-------
float
Calculated global weight.
"""
if global_scheme == "n":
return 1.0
elif global_scheme == "f":
return np.log2(1.0 * totaldocs / docfreq)
elif global_scheme == "t":
return np.log2((totaldocs + 1.0) / docfreq)
elif global_scheme == "p":
return max(0, np.log2((1.0 * totaldocs - docfreq) / docfreq))
@deprecated("Function will be removed in 4.0.0")
def smartirs_normalize(x, norm_scheme, return_norm=False):
"""Normalize a vector using the normalization scheme specified in `norm_scheme`.
Parameters
----------
x : numpy.ndarray
The tf-idf vector.
norm_scheme : {'n', 'c'}
Document length normalization scheme.
return_norm : bool, optional
Return the length of `x` as well?
Returns
-------
numpy.ndarray
Normalized array.
float (only if return_norm is set)
Norm of `x`.
"""
if norm_scheme == "n":
if return_norm:
_, length = matutils.unitvec(x, return_norm=return_norm)
return x, length
else:
return x
elif norm_scheme == "c":
return matutils.unitvec(x, return_norm=return_norm)
class TfidfModel(interfaces.TransformationABC):
"""Objects of this class realize the transformation between word-document co-occurrence matrix (int)
into a locally/globally weighted TF-IDF matrix (positive floats).
Examples
--------
.. sourcecode:: pycon
>>> import gensim.downloader as api
>>> from gensim.models import TfidfModel
>>> from gensim.corpora import Dictionary
>>>
>>> dataset = api.load("text8")
>>> dct = Dictionary(dataset) # fit dictionary
>>> corpus = [dct.doc2bow(line) for line in dataset] # convert corpus to BoW format
>>>
>>> model = TfidfModel(corpus) # fit model
>>> vector = model[corpus[0]] # apply model to the first corpus document
"""
def __init__(self, corpus=None, id2word=None, dictionary=None, wlocal=utils.identity,
wglobal=df2idf, normalize=True, smartirs=None, pivot=None, slope=0.25):
r"""Compute TF-IDF by multiplying a local component (term frequency) with a global component
(inverse document frequency), and normalizing the resulting documents to unit length.
Formula for non-normalized weight of term :math:`i` in document :math:`j` in a corpus of :math:`D` documents
.. math:: weight_{i,j} = frequency_{i,j} * log_2 \frac{D}{document\_freq_{i}}
or, more generally
.. math:: weight_{i,j} = wlocal(frequency_{i,j}) * wglobal(document\_freq_{i}, D)
so you can plug in your own custom :math:`wlocal` and :math:`wglobal` functions.
Parameters
----------
corpus : iterable of iterable of (int, int), optional
Input corpus
id2word : {dict, :class:`~gensim.corpora.Dictionary`}, optional
Mapping token - id, that was used for converting input data to bag of words format.
dictionary : :class:`~gensim.corpora.Dictionary`
If `dictionary` is specified, it must be a `corpora.Dictionary` object and it will be used.
to directly construct the inverse document frequency mapping (then `corpus`, if specified, is ignored).
wlocals : callable, optional
Function for local weighting, default for `wlocal` is :func:`~gensim.utils.identity`
(other options: :func:`numpy.sqrt`, `lambda tf: 0.5 + (0.5 * tf / tf.max())`, etc.).
wglobal : callable, optional
Function for global weighting, default is :func:`~gensim.models.tfidfmodel.df2idf`.
normalize : {bool, callable}, optional
Normalize document vectors to unit euclidean length? You can also inject your own function into `normalize`.
smartirs : str, optional
SMART (System for the Mechanical Analysis and Retrieval of Text) Information Retrieval System,
a mnemonic scheme for denoting tf-idf weighting variants in the vector space model.
The mnemonic for representing a combination of weights takes the form XYZ,
for example 'ntc', 'bpn' and so on, where the letters represents the term weighting of the document vector.
Term frequency weighing:
* `b` - binary,
* `t` or `n` - raw,
* `a` - augmented,
* `l` - logarithm,
* `d` - double logarithm,
* `L` - log average.
Document frequency weighting:
* `x` or `n` - none,
* `f` - idf,
* `t` - zero-corrected idf,
* `p` - probabilistic idf.
Document normalization:
* `x` or `n` - none,
* `c` - cosine,
* `u` - pivoted unique,
* `b` - pivoted character length.
Default is 'nfc'.
For more information visit `SMART Information Retrieval System
`_.
pivot : float or None, optional
In information retrieval, TF-IDF is biased against long documents [1]_. Pivoted document length
normalization solves this problem by changing the norm of a document to `slope * old_norm + (1.0 -
slope) * pivot`.
You can either set the `pivot` by hand, or you can let Gensim figure it out automatically with the following
two steps:
* Set either the `u` or `b` document normalization in the `smartirs` parameter.
* Set either the `corpus` or `dictionary` parameter. The `pivot` will be automatically determined from
the properties of the `corpus` or `dictionary`.
If `pivot` is None and you don't follow steps 1 and 2, then pivoted document length normalization will be
disabled. Default is None.
See also the blog post at https://rare-technologies.com/pivoted-document-length-normalisation/.
slope : float, optional
In information retrieval, TF-IDF is biased against long documents [1]_. Pivoted document length
normalization solves this problem by changing the norm of a document to `slope * old_norm + (1.0 -
slope) * pivot`.
Setting the `slope` to 0.0 uses only the `pivot` as the norm, and setting the `slope` to 1.0 effectively
disables pivoted document length normalization. Singhal [2]_ suggests setting the `slope` between 0.2 and
0.3 for best results. Default is 0.25.
See also the blog post at https://rare-technologies.com/pivoted-document-length-normalisation/.
References
----------
.. [1] Singhal, A., Buckley, C., & Mitra, M. (1996). `Pivoted Document Length
Normalization `_. *SIGIR Forum*, 51, 176–184.
.. [2] Singhal, A. (2001). `Modern information retrieval: A brief overview `_.
*IEEE Data Eng. Bull.*, 24(4), 35–43.
"""
self.id2word = id2word
self.wlocal, self.wglobal, self.normalize = wlocal, wglobal, normalize
self.num_docs, self.num_nnz, self.idfs = None, None, None
self.smartirs = resolve_weights(smartirs) if smartirs is not None else None
self.slope = slope
self.pivot = pivot
self.eps = 1e-12
if smartirs:
n_tf, n_df, n_n = self.smartirs
self.wlocal = partial(smartirs_wlocal, local_scheme=n_tf)
self.wglobal = partial(smartirs_wglobal, global_scheme=n_df)
if dictionary:
# user supplied a Dictionary object, which already contains all the
# statistics we need to construct the IDF mapping. we can skip the
# step that goes through the corpus (= an optimization).
if corpus:
logger.warning(
"constructor received both corpus and explicit inverse document frequencies; ignoring the corpus"
)
self.num_docs, self.num_nnz = dictionary.num_docs, dictionary.num_nnz
self.cfs = dictionary.cfs.copy()
self.dfs = dictionary.dfs.copy()
self.term_lens = {termid: len(term) for termid, term in dictionary.items()}
self.idfs = precompute_idfs(self.wglobal, self.dfs, self.num_docs)
if not id2word:
self.id2word = dictionary
elif corpus:
self.initialize(corpus)
else:
# NOTE: everything is left uninitialized; presumably the model will
# be initialized in some other way
pass
# If smartirs is not None, override pivot and normalize
if not smartirs:
return
if self.pivot is not None:
if n_n in 'ub':
logger.warning("constructor received pivot; ignoring smartirs[2]")
return
if n_n in 'ub' and callable(self.normalize):
logger.warning("constructor received smartirs; ignoring normalize")
if n_n in 'ub' and not dictionary and not corpus:
logger.warning("constructor received no corpus or dictionary; ignoring smartirs[2]")
elif n_n == "u":
self.pivot = 1.0 * self.num_nnz / self.num_docs
elif n_n == "b":
self.pivot = 1.0 * sum(
self.cfs[termid] * (self.term_lens[termid] + 1.0) for termid in dictionary.keys()
) / self.num_docs
@classmethod
def load(cls, *args, **kwargs):
"""Load a previously saved TfidfModel class. Handles backwards compatibility from
older TfidfModel versions which did not use pivoted document normalization.
"""
model = super(TfidfModel, cls).load(*args, **kwargs)
if not hasattr(model, 'pivot'):
model.pivot = None
logger.info('older version of %s loaded without pivot arg', cls.__name__)
logger.info('Setting pivot to %s.', model.pivot)
if not hasattr(model, 'slope'):
model.slope = 0.65
logger.info('older version of %s loaded without slope arg', cls.__name__)
logger.info('Setting slope to %s.', model.slope)
if not hasattr(model, 'smartirs'):
model.smartirs = None
logger.info('older version of %s loaded without smartirs arg', cls.__name__)
logger.info('Setting smartirs to %s.', model.smartirs)
return model
def __str__(self):
return "%s" % (self.__class__.__name__, self.num_docs, self.num_nnz)
def initialize(self, corpus):
"""Compute inverse document weights, which will be used to modify term frequencies for documents.
Parameters
----------
corpus : iterable of iterable of (int, int)
Input corpus.
"""
logger.info("collecting document frequencies")
dfs = {}
numnnz, docno = 0, -1
for docno, bow in enumerate(corpus):
if docno % 10000 == 0:
logger.info("PROGRESS: processing document #%i", docno)
numnnz += len(bow)
for termid, _ in bow:
dfs[termid] = dfs.get(termid, 0) + 1
# keep some stats about the training corpus
self.num_docs = docno + 1
self.num_nnz = numnnz
self.cfs = None
self.dfs = dfs
self.term_lengths = None
# and finally compute the idf weights
self.idfs = precompute_idfs(self.wglobal, self.dfs, self.num_docs)
self.add_lifecycle_event(
"initialize",
msg=(
f"calculated IDF weights for {self.num_docs} documents and {max(dfs.keys()) + 1 if dfs else 0}"
f" features ({self.num_nnz} matrix non-zeros)"
),
)
def __getitem__(self, bow, eps=1e-12):
"""Get the tf-idf representation of an input vector and/or corpus.
bow : {list of (int, int), iterable of iterable of (int, int)}
Input document in the `sparse Gensim bag-of-words format
`_,
or a streamed corpus of such documents.
eps : float
Threshold value, will remove all position that have tfidf-value less than `eps`.
Returns
-------
vector : list of (int, float)
TfIdf vector, if `bow` is a single document
:class:`~gensim.interfaces.TransformedCorpus`
TfIdf corpus, if `bow` is a corpus.
"""
self.eps = eps
# if the input vector is in fact a corpus, return a transformed corpus as a result
is_corpus, bow = utils.is_corpus(bow)
if is_corpus:
return self._apply(bow)
# unknown (new) terms will be given zero weight (NOT infinity/huge weight,
# as strict application of the IDF formula would dictate)
termid_array, tf_array = [], []
for termid, tf in bow:
termid_array.append(termid)
tf_array.append(tf)
tf_array = self.wlocal(np.array(tf_array))
vector = [
(termid, tf * self.idfs.get(termid))
for termid, tf in zip(termid_array, tf_array) if abs(self.idfs.get(termid, 0.0)) > self.eps
]
# and finally, normalize the vector either to unit length, or use a
# user-defined normalization function
if self.smartirs:
n_n = self.smartirs[2]
if n_n == "n" or (n_n in 'ub' and self.pivot is None):
if self.pivot is not None:
_, old_norm = matutils.unitvec(vector, return_norm=True)
norm_vector = vector
elif n_n == "c":
if self.pivot is not None:
_, old_norm = matutils.unitvec(vector, return_norm=True)
else:
norm_vector = matutils.unitvec(vector)
elif n_n == "u":
_, old_norm = matutils.unitvec(vector, return_norm=True, norm='unique')
elif n_n == "b":
old_norm = sum(freq * (self.term_lens[termid] + 1.0) for termid, freq in bow)
else:
if self.normalize is True:
self.normalize = matutils.unitvec
elif self.normalize is False:
self.normalize = utils.identity
if self.pivot is not None:
_, old_norm = self.normalize(vector, return_norm=True)
else:
norm_vector = self.normalize(vector)
if self.pivot is None:
norm_vector = [(termid, weight) for termid, weight in norm_vector if abs(weight) > self.eps]
else:
pivoted_norm = (1 - self.slope) * self.pivot + self.slope * old_norm
norm_vector = [
(termid, weight / float(pivoted_norm))
for termid, weight in vector
if abs(weight / float(pivoted_norm)) > self.eps
]
return norm_vector