EVOLUTION-MANAGER

Edit File: ngrams.py

# Copyright 2007 Matt Chaput. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY MATT CHAPUT ``AS IS'' AND ANY EXPRESS OR # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO # EVENT SHALL MATT CHAPUT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, # OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # The views and conclusions contained in the software and documentation are # those of the authors and should not be interpreted as representing official # policies, either expressed or implied, of Matt Chaput. from whoosh.compat import text_type from whoosh.compat import xrange from whoosh.analysis.acore import Token from whoosh.analysis.filters import Filter, LowercaseFilter from whoosh.analysis.tokenizers import Tokenizer, RegexTokenizer # Tokenizer class NgramTokenizer(Tokenizer): """Splits input text into N-grams instead of words. >>> ngt = NgramTokenizer(4) >>> [token.text for token in ngt("hi there")] ["hi t", "i th", " the", "ther", "here"] Note that this tokenizer does NOT use a regular expression to extract words, so the grams emitted by it will contain whitespace, punctuation, etc. You may want to massage the input or add a custom filter to this tokenizer's output. Alternatively, if you only want sub-word grams without whitespace, you could combine a RegexTokenizer with NgramFilter instead. """ __inittypes__ = dict(minsize=int, maxsize=int) def __init__(self, minsize, maxsize=None): """ :param minsize: The minimum size of the N-grams. :param maxsize: The maximum size of the N-grams. If you omit this parameter, maxsize == minsize. """ self.min = minsize self.max = maxsize or minsize def __eq__(self, other): if self.__class__ is other.__class__: if self.min == other.min and self.max == other.max: return True return False def __call__(self, value, positions=False, chars=False, keeporiginal=False, removestops=True, start_pos=0, start_char=0, mode='', **kwargs): assert isinstance(value, text_type), "%r is not unicode" % value inlen = len(value) t = Token(positions, chars, removestops=removestops, mode=mode) pos = start_pos if mode == "query": size = min(self.max, inlen) for start in xrange(0, inlen - size + 1): end = start + size if end > inlen: continue t.text = value[start:end] if keeporiginal: t.original = t.text t.stopped = False if positions: t.pos = pos if chars: t.startchar = start_char + start t.endchar = start_char + end yield t pos += 1 else: for start in xrange(0, inlen - self.min + 1): for size in xrange(self.min, self.max + 1): end = start + size if end > inlen: continue t.text = value[start:end] if keeporiginal: t.original = t.text t.stopped = False if positions: t.pos = pos if chars: t.startchar = start_char + start t.endchar = start_char + end yield t pos += 1 # Filter class NgramFilter(Filter): """Splits token text into N-grams. >>> rext = RegexTokenizer() >>> stream = rext("hello there") >>> ngf = NgramFilter(4) >>> [token.text for token in ngf(stream)] ["hell", "ello", "ther", "here"] """ __inittypes__ = dict(minsize=int, maxsize=int) def __init__(self, minsize, maxsize=None, at=None): """ :param minsize: The minimum size of the N-grams. :param maxsize: The maximum size of the N-grams. If you omit this parameter, maxsize == minsize. :param at: If 'start', only take N-grams from the start of each word. if 'end', only take N-grams from the end of each word. Otherwise, take all N-grams from the word (the default). """ self.min = minsize self.max = maxsize or minsize self.at = 0 if at == "start": self.at = -1 elif at == "end": self.at = 1 def __eq__(self, other): return other and self.__class__ is other.__class__\ and self.min == other.min and self.max == other.max def __call__(self, tokens): assert hasattr(tokens, "__iter__") at = self.at for t in tokens: text = t.text if len(text) < self.min: continue chars = t.chars if chars: startchar = t.startchar # Token positions don't mean much for N-grams, # so we'll leave the token's original position # untouched. if t.mode == "query": size = min(self.max, len(t.text)) if at == -1: t.text = text[:size] if chars: t.endchar = startchar + size yield t elif at == 1: t.text = text[0 - size:] if chars: t.startchar = t.endchar - size yield t else: for start in xrange(0, len(text) - size + 1): t.text = text[start:start + size] if chars: t.startchar = startchar + start t.endchar = startchar + start + size yield t else: if at == -1: limit = min(self.max, len(text)) for size in xrange(self.min, limit + 1): t.text = text[:size] if chars: t.endchar = startchar + size yield t elif at == 1: if chars: original_startchar = t.startchar start = max(0, len(text) - self.max) for i in xrange(start, len(text) - self.min + 1): t.text = text[i:] if chars: t.startchar = original_startchar + i yield t else: for start in xrange(0, len(text) - self.min + 1): for size in xrange(self.min, self.max + 1): end = start + size if end > len(text): continue t.text = text[start:end] if chars: t.startchar = startchar + start t.endchar = startchar + end yield t # Analyzers def NgramAnalyzer(minsize, maxsize=None): """Composes an NgramTokenizer and a LowercaseFilter. >>> ana = NgramAnalyzer(4) >>> [token.text for token in ana("hi there")] ["hi t", "i th", " the", "ther", "here"] """ return NgramTokenizer(minsize, maxsize=maxsize) | LowercaseFilter() def NgramWordAnalyzer(minsize, maxsize=None, tokenizer=None, at=None): if not tokenizer: tokenizer = RegexTokenizer() return tokenizer | LowercaseFilter() | NgramFilter(minsize, maxsize, at=at)