# Copyright 2007 Matt Chaput. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#    1. Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#
#    2. Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY MATT CHAPUT ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
# EVENT SHALL MATT CHAPUT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and documentation are
# those of the authors and should not be interpreted as representing official
# policies, either expressed or implied, of Matt Chaput.

from __future__ import division

import copy
import fnmatch
import re

import engine.matching
import whoosh
from engine.analysis.acore import CylleneusToken
from engine.query import qcore
from whoosh.compat import bytes_type, text_type, u
from whoosh.lang.morph_en import variations


class CylleneusTerm(qcore.Query):
    """Matches documents containing the given term (fieldname+text pair).
    >>> CylleneusTerm("content", u"render")
    """

    __inittypes__ = dict(fieldname=str, text=text_type, boost=float, meta=bool)

    def __init__(self, fieldname, text, boost=1.0, minquality=None, meta=False):
        self.fieldname = fieldname
        self.text = text
        self.boost = boost
        self.minquality = minquality
        self.meta = meta

    def __eq__(self, other):
        return (other
                and self.__class__ is other.__class__
                and self.fieldname == other.fieldname
                and self.text == other.text
                and self.boost == other.boost)

    def __repr__(self):
        r = "%s(%r, %r" % (self.__class__.__name__, self.fieldname, self.text)
        if self.boost != 1.0:
            r += ", boost=%s" % self.boost
        r += ")"
        return r

    def __unicode__(self):
        text = self.text
        if isinstance(text, bytes_type):
            try:
                text = text.decode("ascii")
            except UnicodeDecodeError:
                text = repr(text)

        t = u("%s:%s") % (self.fieldname, text)
        if self.boost != 1:
            t += u("^") + text_type(self.boost)
        return t

    __str__ = __unicode__

    def __hash__(self):
        return hash(self.fieldname) ^ hash(self.text) ^ hash(self.boost)

    def has_terms(self):
        return True

    def tokens(self, boost=1.0):
        yield CylleneusToken(fieldname=self.fieldname, text=self.text,
                    boost=boost * self.boost, startchar=self.startchar,
                    endchar=self.endchar, chars=True)

    def terms(self, phrases=False):
        if self.field():
            yield (self.field(), self.text)

    def replace(self, fieldname, oldtext, newtext):
        q = copy.copy(self)
        if q.fieldname == fieldname and q.text == oldtext:
            q.text = newtext
        return q

    def estimate_size(self, ixreader):
        fieldname = self.fieldname
        if fieldname not in ixreader.schema:
            return 0

        field = ixreader.schema[fieldname]
        try:
            text = field.to_bytes(self.text)
        except ValueError:
            return 0

        return ixreader.doc_frequency(fieldname, text)

    def matcher(self, searcher, context=None):
        fieldname = self.fieldname
        text = self.text
        if fieldname not in searcher.schema:
            return engine.matching.mcore.NullMatcher()

        field = searcher.schema[fieldname]
        try:
            text = field.to_bytes(text)
        except ValueError:
            return engine.matching.mcore.NullMatcher()

        if (self.fieldname, text) in searcher.reader():
            if context is None:
                w = searcher.weighting
            else:
                w = context.weighting

            m = searcher.postings(self.fieldname, text, weighting=w)
            if self.minquality:
                m.set_min_quality(self.minquality)
            if self.boost != 1.0:
                m = engine.matching.wrappers.WrappingMatcher(m, boost=self.boost)
            return m
        else:
            return engine.matching.mcore.NullMatcher()

class Form(CylleneusTerm):
    fieldname = 'form'

    __inittypes__ = dict(fieldname=str, text=whoosh.compat.text_type, boost=float, annotation=CylleneusTerm, meta=bool)

    def __init__(self, fieldname, text, boost=1.0, minquality=None, annotation=None, meta=False):
        super(Form, self).__init__(fieldname, text, boost=1.0, minquality=None)
        self.fieldname = fieldname
        self.text = text
        self.boost = boost
        self.minquality = minquality
        self.annotation = annotation
        self.meta = meta

    def __eq__(self, other):
        return (other
                and self.__class__ is other.__class__
                and self.fieldname == other.fieldname
                and self.text == other.text
                and self.annotation == other.annotation
                and self.boost == other.boost)

    def __repr__(self):
        r = "%s(%r, %r" % (self.__class__.__name__, self.fieldname, self.text)
        if self.boost != 1.0:
            r += ", boost=%s" % self.boost
        if self.annotation:
            r += ", annotation=%s" % self.annotation
        r += ")"
        return r

    def __unicode__(self):
        text = self.text
        if isinstance(text, whoosh.compat.bytes_type):
            try:
                text = text.decode("ascii")
            except UnicodeDecodeError:
                text = repr(text)

        t = whoosh.compat.u("'%s'") % text
        # if self.boost != 1:
        #     t += whoosh.compat.u("^") + whoosh.compat.text_type(self.boost)
        return t

    __str__ = __unicode__

    def __hash__(self):
        return hash(self.fieldname) ^ hash(self.text) ^ hash(self.boost)

    def has_terms(self):
        return True

    def tokens(self, boost=1.0):
        yield CylleneusToken(fieldname=self.fieldname, text=self.text,
                    boost=boost * self.boost, startchar=self.startchar,
                    endchar=self.endchar, chars=True, annotation=self.annotation)

    def terms(self, phrases=False):
        if self.field():
            yield (self.field(), self.text)

    def replace(self, fieldname, oldtext, newtext):
        q = copy.copy(self)
        if q.fieldname == fieldname and q.text == oldtext:
            q.text = newtext
        return q

    def estimate_size(self, ixreader):
        fieldname = self.fieldname
        if fieldname not in ixreader.schema:
            return 0

        field = ixreader.schema[fieldname]
        try:
            text = field.to_bytes(self.text)
        except ValueError:
            return 0

        return ixreader.doc_frequency(fieldname, text)

    def matcher(self, searcher, context=None):
        fieldname = self.fieldname
        text = self.text
        if fieldname not in searcher.schema:
            return engine.matching.mcore.NullMatcher()

        field = searcher.schema[fieldname]
        try:
            text = field.to_bytes(text)
        except ValueError:
            return engine.matching.mcore.NullMatcher()

        if (self.fieldname, text) in searcher.reader():
            if context is None:
                w = searcher.weighting
            else:
                w = context.weighting

            m = searcher.postings(self.fieldname, text, weighting=w)
            if self.minquality:
                m.set_min_quality(self.minquality)
            if self.boost != 1.0:
                m = engine.matching.wrappers.WrappingMatcher(m, boost=self.boost)
            return m
        else:
            return engine.matching.mcore.NullMatcher()

    def __iter__(self):
        return iter([self,])


class Lemma(CylleneusTerm):
    fieldname = 'lemma'

    __inittypes__ = dict(fieldname=str, text=whoosh.compat.text_type, boost=float, annotation=CylleneusTerm, meta=bool)

    def __init__(self, fieldname, text, boost=1.0, minquality=None, annotation=None, meta=False):
        super(Lemma, self).__init__(fieldname, text, boost=1.0, minquality=None)
        self.fieldname = fieldname
        self.boost = boost
        self.text = text
        self.minquality = minquality
        self.annotation = annotation
        self.meta = meta

    def __eq__(self, other):
        return (other
                and self.__class__ is other.__class__
                and self.fieldname == other.fieldname
                and self.text == other.text
                and self.annotation == self.annotation
                and self.boost == other.boost)

    def __repr__(self):
        r = "%s(%r, %r" % (self.__class__.__name__, self.fieldname, self.text)
        if self.boost != 1.0:
            r += ", boost=%s" % self.boost
        if self.annotation:
            r += ", annotation=%s" % self.annotation
        r += ")"
        return r

    def __unicode__(self):
        text = self.text
        if isinstance(text, whoosh.compat.bytes_type):
            try:
                text = text.decode("ascii")
            except UnicodeDecodeError:
                text = repr(text)

        t = whoosh.compat.u("<%s>") % text
        if self.annotation:
            t += whoosh.compat.u(":") + whoosh.compat.text_type(self.annotation)
        # if self.boost != 1:
        #     t += whoosh.compat.u("^") + whoosh.compat.text_type(self.boost)
        return t

    __str__ = __unicode__

    def __hash__(self):
        return hash(self.fieldname) ^ hash(self.text) ^ hash(self.boost)

    def has_terms(self):
        return True

    def tokens(self, boost=1.0):
        kwargs = {}
        if hasattr(self, 'pos'):
            kwargs["pos"] = self.pos
        yield CylleneusToken(fieldname=self.fieldname, text=self.text,
                    boost=boost * self.boost, startchar=self.startchar,
                    endchar=self.endchar, chars=True, **kwargs)

    def terms(self, phrases=False):
        if self.field():
            yield (self.field(), self.text)

    def replace(self, fieldname, oldtext, newtext):
        q = copy.copy(self)
        if q.fieldname == fieldname and q.text == oldtext:
            q.text = newtext
        return q

    def estimate_size(self, ixreader):
        fieldname = self.fieldname
        if fieldname not in ixreader.schema:
            return 0

        field = ixreader.schema[fieldname]
        try:
            text = field.to_bytes(self.text)
        except ValueError:
            return 0

        return ixreader.doc_frequency(fieldname, text)

    def matcher(self, searcher, context=None):
        fieldname = self.fieldname
        text = self.text
        if fieldname not in searcher.schema:
            return engine.matching.mcore.NullMatcher()

        field = searcher.schema[fieldname]
        try:
            text = field.to_bytes(text)
        except ValueError:
            return engine.matching.mcore.NullMatcher()

        if (self.fieldname, text) in searcher.reader():
            if context is None:
                w = searcher.weighting
            else:
                w = context.weighting

            m = searcher.postings(self.fieldname, text, weighting=w)
            if self.minquality:
                m.set_min_quality(self.minquality)
            if self.boost != 1.0:
                m = engine.matching.wrappers.WrappingMatcher(m, boost=self.boost)
            return m
        else:
            return engine.matching.mcore.NullMatcher()

    def __iter__(self):
        return iter([self,])


class Semfield(CylleneusTerm):
    """Matches documents containing the given term (fieldname+semfield)."""

    fieldname = 'semfield'

    __inittypes__ = dict(fieldname=str, text=whoosh.compat.text_type, boost=float, annotation=CylleneusTerm, meta=bool)

    def __init__(self, fieldname, text, boost=1.0, minquality=None, annotation=None, meta=False):
        super(Semfield, self).__init__(fieldname, text, boost=1.0, minquality=None)
        self.fieldname = 'semfield'
        self.text = text
        self.boost = boost
        self.minquality = minquality
        self.annotation = annotation
        self.meta = meta

    def __eq__(self, other):
        return (other
                and self.__class__ is other.__class__
                and self.fieldname == other.fieldname
                and self.text == other.text
                and self.boost == other.boost)

    def __repr__(self):
        r = "%s(%r, %r" % (self.__class__.__name__, self.fieldname, self.text)
        if self.boost != 1.0:
            r += ", boost=%s" % self.boost
        if self.annotation:
            r += ", annotation=%s" % self.annotation
        r += ")"
        return r

    def __unicode__(self):
        text = self.text
        if isinstance(text, whoosh.compat.bytes_type):
            try:
                text = text.decode("ascii")
            except UnicodeDecodeError:
                text = repr(text)

        t = whoosh.compat.u("{%s}") % text
        if self.annotation:
            t += whoosh.compat.u(":") + whoosh.compat.text_type(self.annotation)
        # if self.boost != 1:
        #     t += whoosh.compat.u("^") + whoosh.compat.text_type(self.boost)
        return t

    __str__ = __unicode__

    def __hash__(self):
        return hash(self.fieldname) ^ hash(self.text) ^ hash(self.boost)

    def has_terms(self):
        return True

    def tokens(self, boost=1.0):
        yield CylleneusToken(fieldname=self.fieldname, text=self.text,
                    boost=boost * self.boost, startchar=self.startchar,
                    endchar=self.endchar, chars=True)

    def terms(self, phrases=False):
        if self.field():
            yield (self.field(), self.text)

    def replace(self, fieldname, oldtext, newtext):
        q = copy.copy(self)
        if q.fieldname == fieldname and q.text == oldtext:
            q.text = newtext
        return q

    def estimate_size(self, ixreader):
        fieldname = self.fieldname
        if fieldname not in ixreader.schema:
            return 0

        field = ixreader.schema[fieldname]
        try:
            text = field.to_bytes(self.text)
        except ValueError:
            return 0

        return ixreader.doc_frequency(fieldname, text)

    def matcher(self, searcher, context=None):
        fieldname = self.fieldname
        text = self.text
        if fieldname not in searcher.schema:
            return engine.matching.mcore.NullMatcher()

        field = searcher.schema[fieldname]
        try:
            text = field.to_bytes(text)
        except ValueError:
            return engine.matching.mcore.NullMatcher()

        if (self.fieldname, text) in searcher.reader():
            if context is None:
                w = searcher.weighting
            else:
                w = context.weighting

            m = searcher.postings(self.fieldname, text, weighting=w)
            if self.minquality:
                m.set_min_quality(self.minquality)
            if self.boost != 1.0:
                m = engine.matching.wrappers.WrappingMatcher(m, boost=self.boost)
            return m
        else:
            return engine.matching.mcore.NullMatcher()

    def __iter__(self):
        return iter([self,])

class Gloss(CylleneusTerm):
    """Matches documents containing the given term (fieldname+gloss)."""

    fieldname = 'synset'

    __inittypes__ = dict(fieldname=str, text=whoosh.compat.text_type, boost=float, annotation=CylleneusTerm, meta=bool)

    def __init__(self, fieldname, text, boost=1.0, minquality=None, annotation=None, meta=False):
        super(Gloss, self).__init__(fieldname, text, boost=1.0, minquality=None)
        self.fieldname = 'synset'
        self.text = text
        self.boost = boost
        self.minquality = minquality
        self.annotation = annotation
        self.meta = meta

    def __eq__(self, other):
        return (other
                and self.__class__ is other.__class__
                and self.fieldname == other.fieldname
                and self.text == other.text
                and self.boost == other.boost)

    def __repr__(self):
        r = "%s(%r, %r" % (self.__class__.__name__, self.fieldname, self.text)
        if self.boost != 1.0:
            r += ", boost=%s" % self.boost
        if self.annotation:
            r += ", annotation=%s" % self.annotation
        r += ")"
        return r

    def __unicode__(self):
        text = self.text
        if isinstance(text, whoosh.compat.bytes_type):
            try:
                text = text.decode("ascii")
            except UnicodeDecodeError:
                text = repr(text)

        t = whoosh.compat.u("[%s]") % text
        if self.annotation:
            t += whoosh.compat.u(":") + whoosh.compat.text_type(self.annotation)
        # if self.boost != 1:
        #     t += whoosh.compat.u("^") + whoosh.compat.text_type(self.boost)
        return t

    __str__ = __unicode__

    def __hash__(self):
        return hash(self.fieldname) ^ hash(self.text) ^ hash(self.boost)

    def has_terms(self):
        return True

    def tokens(self, boost=1.0):
        yield CylleneusToken(fieldname=self.fieldname, text=self.text,
                    boost=boost * self.boost, startchar=self.startchar,
                    endchar=self.endchar, chars=True)

    def terms(self, phrases=False):
        if self.field():
            yield (self.field(), self.text)

    def replace(self, fieldname, oldtext, newtext):
        q = copy.copy(self)
        if q.fieldname == fieldname and q.text == oldtext:
            q.text = newtext
        return q

    def estimate_size(self, ixreader):
        fieldname = self.fieldname
        if fieldname not in ixreader.schema:
            return 0

        field = ixreader.schema[fieldname]
        try:
            text = field.to_bytes(self.text)
        except ValueError:
            return 0

        return ixreader.doc_frequency(fieldname, text)

    def matcher(self, searcher, context=None):
        fieldname = self.fieldname
        text = self.text
        if fieldname not in searcher.schema:
            return engine.matching.mcore.NullMatcher()

        field = searcher.schema[fieldname]
        try:
            text = field.to_bytes(text)
        except ValueError:
            return engine.matching.mcore.NullMatcher()

        if (self.fieldname, text) in searcher.reader():
            if context is None:
                w = searcher.weighting
            else:
                w = context.weighting

            m = searcher.postings(self.fieldname, text, weighting=w)
            if self.minquality:
                m.set_min_quality(self.minquality)
            if self.boost != 1.0:
                m = engine.matching.wrappers.WrappingMatcher(m, boost=self.boost)
            return m
        else:
            return engine.matching.mcore.NullMatcher()

    def __iter__(self):
        return iter([self,])


class Annotation(CylleneusTerm):
    """Matches documents containing the given term (fieldname+annotation)."""

    fieldname = 'annotation'

    __inittypes__ = dict(fieldname=str, text=whoosh.compat.text_type, boost=float, annotation=qcore.Query, meta=bool)

    def __init__(self, fieldname, text, boost=1.0, minquality=None, annotation=None, meta=False):
        super(Annotation, self).__init__(fieldname, text, boost=1.0, minquality=None)
        self.fieldname = fieldname
        self.text = text
        self.boost = boost
        self.minquality = minquality
        self.annotation = annotation
        self.meta = meta

    def __eq__(self, other):
        return (other
                and self.__class__ is other.__class__
                and self.fieldname == other.fieldname
                and self.text == other.text
                and self.boost == other.boost)

    def __repr__(self):
        r = "%s(%r, %r" % (self.__class__.__name__, self.fieldname, self.text)
        if self.boost != 1.0:
            r += ", boost=%s" % self.boost
        r += ")"
        return r

    def __unicode__(self):
        text = self.text
        if isinstance(text, whoosh.compat.bytes_type):
            try:
                text = text.decode("ascii")
            except UnicodeDecodeError:
                text = repr(text)

        t = whoosh.compat.u(":%s") % text
        # if self.boost != 1:
        #     t += whoosh.compat.u("^") + whoosh.compat.text_type(self.boost)
        return t

    __str__ = __unicode__

    def __hash__(self):
        return hash(self.fieldname) ^ hash(self.text) ^ hash(self.boost)

    def has_terms(self):
        return True

    def tokens(self, boost=1.0):
        yield CylleneusToken(fieldname=self.fieldname, text=self.text,
                    boost=boost * self.boost, startchar=self.startchar,
                    endchar=self.endchar, chars=True)

    def terms(self, phrases=False):
        if self.field():
            yield (self.field(), self.text)

    def replace(self, fieldname, oldtext, newtext):
        q = copy.copy(self)
        if q.fieldname == fieldname and q.text == oldtext:
            q.text = newtext
        return q

    def estimate_size(self, ixreader):
        fieldname = self.fieldname
        if fieldname not in ixreader.schema:
            return 0

        field = ixreader.schema[fieldname]
        try:
            text = field.to_bytes(self.text)
        except ValueError:
            return 0

        return ixreader.doc_frequency(fieldname, text)

    def matcher(self, searcher, context=None):
        fieldname = self.fieldname
        text = self.text
        if fieldname not in searcher.schema:
            return engine.matching.mcore.NullMatcher()

        field = searcher.schema[fieldname]
        try:
            text = field.to_bytes(text)
        except ValueError:
            return engine.matching.mcore.NullMatcher()

        if (self.fieldname, text) in searcher.reader():
            if context is None:
                w = searcher.weighting
            else:
                w = context.weighting

            m = searcher.postings(self.fieldname, text, weighting=w)
            if self.minquality:
                m.set_min_quality(self.minquality)
            if self.boost != 1.0:
                m = engine.matching.wrappers.WrappingMatcher(m, boost=self.boost)
            return m
        else:
            return engine.matching.mcore.NullMatcher()

    def __iter__(self):
        return iter([self,])


class Morphosyntax(CylleneusTerm):
    """Matches documents containing the given term (fieldname+annotation)."""

    fieldname = 'morphosyntax'

    __inittypes__ = dict(fieldname=str, text=whoosh.compat.text_type, boost=float, annotation=qcore.Query, meta=bool)

    def __init__(self, fieldname, text, boost=1.0, minquality=None, annotation=None, meta=False):
        super(Morphosyntax, self).__init__(fieldname, text, boost=1.0, minquality=None)
        self.fieldname = fieldname
        self.text = text
        self.boost = boost
        self.minquality = minquality
        self.annotation = annotation
        self.meta = meta

    def __eq__(self, other):
        return (other
                and self.__class__ is other.__class__
                and self.fieldname == other.fieldname
                and self.text == other.text
                and self.boost == other.boost)

    def __repr__(self):
        r = "%s(%r, %r" % (self.__class__.__name__, self.fieldname, self.text)
        if self.boost != 1.0:
            r += ", boost=%s" % self.boost
        r += ")"
        return r

    def __unicode__(self):
        text = self.text
        if isinstance(text, whoosh.compat.bytes_type):
            try:
                text = text.decode("ascii")
            except UnicodeDecodeError:
                text = repr(text)

        t = whoosh.compat.u("%s:%s") % (self.fieldname, text)
        if self.boost != 1:
            t += whoosh.compat.u("^") + whoosh.compat.text_type(self.boost)
        return t

    __str__ = __unicode__

    def __hash__(self):
        return hash(self.fieldname) ^ hash(self.text) ^ hash(self.boost)

    def has_terms(self):
        return True

    def tokens(self, boost=1.0):
        yield CylleneusToken(fieldname=self.fieldname, text=self.text,
                    boost=boost * self.boost, startchar=self.startchar,
                    endchar=self.endchar, chars=True)

    def terms(self, phrases=False):
        if self.field():
            yield (self.field(), self.text)

    def replace(self, fieldname, oldtext, newtext):
        q = copy.copy(self)
        if q.fieldname == fieldname and q.text == oldtext:
            q.text = newtext
        return q

    def estimate_size(self, ixreader):
        fieldname = self.fieldname
        if fieldname not in ixreader.schema:
            return 0

        field = ixreader.schema[fieldname]
        try:
            text = field.to_bytes(self.text)
        except ValueError:
            return 0

        return ixreader.doc_frequency(fieldname, text)

    def matcher(self, searcher, context=None):
        fieldname = self.fieldname
        text = self.text
        if fieldname not in searcher.schema:
            return engine.matching.mcore.NullMatcher()

        field = searcher.schema[fieldname]
        try:
            text = field.to_bytes(text)
        except ValueError:
            return engine.matching.mcore.NullMatcher()

        if (self.fieldname, text) in searcher.reader():
            if context is None:
                w = searcher.weighting
            else:
                w = context.weighting

            m = searcher.postings(self.fieldname, text, weighting=w)
            if self.minquality:
                m.set_min_quality(self.minquality)
            if self.boost != 1.0:
                m = engine.matching.wrappers.WrappingMatcher(m, boost=self.boost)
            return m
        else:
            return engine.matching.mcore.NullMatcher()

    def __iter__(self):
        return iter([self,])


class MultiTerm(qcore.Query):
    """Abstract base class for queries that operate on multiple terms in the
    same field.
    """

    constantscore = False

    def _btexts(self, ixreader):
        raise NotImplementedError(self.__class__.__name__)

    def expanded_terms(self, ixreader, phrases=False):
        fieldname = self.field()
        if fieldname:
            for btext in self._btexts(ixreader):
                yield (fieldname, btext)

    def tokens(self, boost=1.0, exreader=None):
        fieldname = self.field()
        if exreader is None:
            btexts = [self.text]
        else:
            btexts = self._btexts(exreader)

        for btext in btexts:
            yield CylleneusToken(fieldname=fieldname, text=btext,
                        boost=boost * self.boost, startchar=self.startchar,
                        endchar=self.endchar, chars=True)

    def simplify(self, ixreader):
        fieldname = self.field()

        if fieldname not in ixreader.schema:
            return qcore.NullQuery()
        field = ixreader.schema[fieldname]

        existing = []
        for btext in sorted(set(self._btexts(ixreader))):
            text = field.from_bytes(btext)
            existing.append(CylleneusTerm(fieldname, text, boost=self.boost))

        if len(existing) == 1:
            return existing[0]
        elif existing:
            from engine.query.compound import Or
            return Or(existing)
        else:
            return qcore.NullQuery

    def estimate_size(self, ixreader):
        fieldname = self.field()
        return sum(ixreader.doc_frequency(fieldname, btext)
                   for btext in self._btexts(ixreader))

    def estimate_min_size(self, ixreader):
        fieldname = self.field()
        return min(ixreader.doc_frequency(fieldname, text)
                   for text in self._btexts(ixreader))

    def matcher(self, searcher, context=None):
        from engine.query.compound import Or

        fieldname = self.field()
        constantscore = self.constantscore

        reader = searcher.reader()
        qs = [CylleneusTerm(fieldname, word) for word in self._btexts(reader)]
        if not qs:
            return engine.matching.mcore.NullMatcher()

        if len(qs) == 1:
            # If there's only one term, just use it
            m = qs[0].matcher(searcher, context)
        else:
            if constantscore:
                # To tell the sub-query that score doesn't matter, set weighting
                # to None
                if context:
                    context = context.set(weighting=None)
                else:
                    from whoosh.searching import SearchContext
                    context = SearchContext(weighting=None)
            # Or the terms together
            m = Or(qs, boost=self.boost).matcher(searcher, context)
        return m


class PatternQuery(MultiTerm):
    """An intermediate base class for common methods of Prefix and Wildcard.
    """

    __inittypes__ = dict(fieldname=str, text=text_type, boost=float, annotation=Annotation, meta=bool)

    def __init__(self, fieldname, text, boost=1.0, constantscore=True, annotation=None, meta=False):
        self.fieldname = fieldname
        self.text = text
        self.boost = boost
        self.constantscore = constantscore
        self.annotation = annotation
        self.meta = meta

    def __eq__(self, other):
        return (other and self.__class__ is other.__class__
                and self.fieldname == other.fieldname
                and self.text == other.text and self.boost == other.boost
                and self.constantscore == other.constantscore)

    def __repr__(self):
        r = "%s(%r, %r" % (self.__class__.__name__, self.fieldname, self.text)
        if self.boost != 1:
            r += ", boost=%s" % self.boost
        r += ")"
        return r

    def __hash__(self):
        return (hash(self.fieldname) ^ hash(self.text) ^ hash(self.boost)
                ^ hash(self.constantscore))

    def _get_pattern(self):
        raise NotImplementedError

    def _find_prefix(self, text):
        # Subclasses/instances should set the SPECIAL_CHARS attribute to a set
        # of characters that mark the end of the literal prefix
        specialchars = self.SPECIAL_CHARS
        i = 0
        for i, char in enumerate(text):
            if char in specialchars:
                break
        return text[:i]

    def _btexts(self, ixreader):
        field = ixreader.schema[self.fieldname]

        exp = re.compile(self._get_pattern())
        prefix = self._find_prefix(self.text)
        if prefix:
            candidates = ixreader.expand_prefix(self.fieldname, prefix)
        else:
            candidates = ixreader.lexicon(self.fieldname)

        from_bytes = field.from_bytes
        for btext in candidates:
            text = from_bytes(btext)
            if exp.match(text):
                yield btext

    def __iter__(self):
        return iter([self,])

    def all_terms(self, ixreader, phrases=True):
        """Returns a set of all terms in this query tree.
        This method exists for backwards-compatibility. Use iter_all_terms()
        instead.
        :param phrases: Whether to add words found in Phrase queries.
        :rtype: set
        """

        return set(self.iter_all_terms(ixreader, phrases=phrases))

    def iter_all_terms(self, ixreader, phrases=True):
        """Returns an iterator of (fieldname, text) pairs for all terms in
        this query tree.
        >>> qp = qparser.QueryParser("text", myindex.schema)
        >>> q = myparser.parse("alfa bravo title:charlie")
        >>> # List the terms in a query
        >>> list(q.iter_all_terms())
        [("text", "alfa"), ("text", "bravo"), ("title", "charlie")]
        >>> # Get a set of all terms in the query that don't exist in the index
        >>> r = myindex.reader()
        >>> missing = set(t for t in q.iter_all_terms() if t not in r)
        set([("text", "alfa"), ("title", "charlie")])
        >>> # All terms in the query that occur in fewer than 5 documents in
        >>> # the index
        >>> [t for t in q.iter_all_terms() if r.doc_frequency(t[0], t[1]) < 5]
        [("title", "charlie")]
        :param phrases: Whether to add words found in Phrase queries.
        """

        for btext in self._btexts(ixreader):
            yield (self.fieldname, u(btext))


class Prefix(PatternQuery):
    """Matches documents that contain any terms that start with the given text.
    >>> # Match documents containing words starting with 'comp'
    >>> Prefix("content", u"comp")
    """

    def __unicode__(self):
        return "%s:%s*" % (self.fieldname, self.text)

    __str__ = __unicode__

    def _btexts(self, ixreader):
        return ixreader.expand_prefix(self.fieldname, self.text)

    def matcher(self, searcher, context=None):
        if self.text == "":
            from engine.query.qcore import Every
            eq = Every(self.fieldname, boost=self.boost)
            return eq.matcher(searcher, context)
        else:
            return PatternQuery.matcher(self, searcher, context)


class Wildcard(PatternQuery):
    """Matches documents that contain any terms that match a "glob" pattern.
    See the Python ``fnmatch`` module for information about globs.
    >>> Wildcard("content", u"in*f?x")
    """

    SPECIAL_CHARS = frozenset("*?[")

    def __unicode__(self):
        return "%s:%s" % (self.fieldname, self.text)

    __str__ = __unicode__

    def _get_pattern(self):
        return fnmatch.translate(self.text)

    def normalize(self):
        # If there are no wildcard characters in this "wildcard", turn it into
        # a simple Form
        text = self.text
        if text == "*":
            from engine.query.qcore import Every
            return Every(self.fieldname, boost=self.boost)
        if "*" not in text and "?" not in text:
            # If no wildcard chars, convert to a normal term.
            return Form(self.fieldname, self.text, boost=self.boost)
        elif ("?" not in text and text.endswith("*")
              and text.find("*") == len(text) - 1):
            # If the only wildcard char is an asterisk at the end, convert to a
            # Prefix query.
            return Prefix(self.fieldname, self.text[:-1], boost=self.boost)
        else:
            return self

    def matcher(self, searcher, context=None):
        if self.text == "*":
            from engine.query.qcore import Every
            eq = Every(self.fieldname, boost=self.boost)
            return eq.matcher(searcher, context)
        else:
            return PatternQuery.matcher(self, searcher, context)

    # _btexts() implemented in PatternQuery


class Regex(PatternQuery):
    """Matches documents that contain any terms that match a regular
    expression. See the Python ``re`` module for information about regular
    expressions.
    """

    SPECIAL_CHARS = frozenset("{}()[].?*+^$\\")

    def __unicode__(self):
        return '%s:r"%s"' % (self.fieldname, self.text)

    __str__ = __unicode__

    def _get_pattern(self):
        return self.text

    def _find_prefix(self, text):
        if "|" in text:
            return ""
        if text.startswith("^"):
            text = text[1:]
        elif text.startswith("\\A"):
            text = text[2:]

        prefix = PatternQuery._find_prefix(self, text)

        lp = len(prefix)
        if lp < len(text) and text[lp] in "*?":
            # we stripped something starting from * or ? - they both MAY mean
            # "0 times". As we had stripped starting from FIRST special char,
            # that implies there were only ordinary chars left of it. Thus,
            # the very last of them is not part of the real prefix:
            prefix = prefix[:-1]
        return prefix

    def matcher(self, searcher, context=None):
        if self.text == ".*":
            from engine.query.qcore import Every
            eq = Every(self.fieldname, boost=self.boost)
            return eq.matcher(searcher, context)
        else:
            return PatternQuery.matcher(self, searcher, context)

    # _btexts() implemented in PatternQuery


class ExpandingTerm(MultiTerm):
    """Intermediate base class for queries such as FuzzyTerm and Variations
    that expand into multiple queries, but come from a single term.
    """

    def has_terms(self):
        return True

    def terms(self, phrases=False):
        if self.field():
            yield (self.field(), self.text)


class FuzzyTerm(ExpandingTerm):
    """Matches documents containing words similar to the given term.
    """

    __inittypes__ = dict(fieldname=str, text=text_type, boost=float,
                         maxdist=float, prefixlength=int)

    def __init__(self, fieldname, text, boost=1.0, maxdist=1,
                 prefixlength=1, constantscore=True):
        """
        :param fieldname: The name of the field to search.
        :param text: The text to search for.
        :param boost: A boost factor to apply to scores of documents matching
            this query.
        :param maxdist: The maximum edit distance from the given text.
        :param prefixlength: The matched terms must share this many initial
            characters with 'text'. For example, if text is "light" and
            prefixlength is 2, then only terms starting with "li" are checked
            for similarity.
        """

        self.fieldname = fieldname
        self.text = text
        self.boost = boost
        self.maxdist = maxdist
        self.prefixlength = prefixlength
        self.constantscore = constantscore

    def __eq__(self, other):
        return (other and self.__class__ is other.__class__
                and self.fieldname == other.fieldname
                and self.text == other.text
                and self.maxdist == other.maxdist
                and self.prefixlength == other.prefixlength
                and self.boost == other.boost
                and self.constantscore == other.constantscore)

    def __repr__(self):
        r = "%s(%r, %r, boost=%f, maxdist=%d, prefixlength=%d)"
        return r % (self.__class__.__name__, self.fieldname, self.text,
                    self.boost, self.maxdist, self.prefixlength)

    def __unicode__(self):
        r = u("%s:%s") % (self.fieldname, self.text) + u("~")
        if self.maxdist > 1:
            r += u("%d") % self.maxdist
        if self.boost != 1.0:
            r += u("^%f") % self.boost
        return r

    __str__ = __unicode__

    def __hash__(self):
        return (hash(self.fieldname) ^ hash(self.text) ^ hash(self.boost)
                ^ hash(self.maxdist) ^ hash(self.prefixlength)
                ^ hash(self.constantscore))

    def _btexts(self, ixreader):
        return ixreader.terms_within(self.fieldname, self.text, self.maxdist,
                                     prefix=self.prefixlength)

    def replace(self, fieldname, oldtext, newtext):
        q = copy.copy(self)
        if q.fieldname == fieldname and q.text == oldtext:
            q.text = newtext
        return q


class Variations(ExpandingTerm):
    """Query that automatically searches for morphological variations of the
    given word in the same field.
    """

    def __init__(self, fieldname, text, boost=1.0):
        self.fieldname = fieldname
        self.text = text
        self.boost = boost

    def __repr__(self):
        r = "%s(%r, %r" % (self.__class__.__name__, self.fieldname, self.text)
        if self.boost != 1:
            r += ", boost=%s" % self.boost
        r += ")"
        return r

    def __eq__(self, other):
        return (other and self.__class__ is other.__class__
                and self.fieldname == other.fieldname
                and self.text == other.text and self.boost == other.boost)

    def __hash__(self):
        return hash(self.fieldname) ^ hash(self.text) ^ hash(self.boost)

    def _btexts(self, ixreader):
        fieldname = self.fieldname
        to_bytes = ixreader.schema[fieldname].to_bytes
        for word in variations(self.text):
            try:
                btext = to_bytes(word)
            except ValueError:
                continue

            if (fieldname, btext) in ixreader:
                yield btext

    def __unicode__(self):
        return u("%s:<%s>") % (self.fieldname, self.text)

    __str__ = __unicode__

    def replace(self, fieldname, oldtext, newtext):
        q = copy.copy(self)
        if q.fieldname == fieldname and q.text == oldtext:
            q.text = newtext
        return q
