store.py

import json
import operator
import os
import re
import shutil
import time
from datetime import datetime, timedelta
from io import BytesIO
from threading import ThreadError

import ipaddress
import six
from cachetools.func import ttl_cache
from redis_collections import Dict, Set
from whoosh.fields import ID, KEYWORD, NGRAMWORDS, Schema
from whoosh.filedb.filestore import FileStorage
from whoosh.qparser import MultifieldParser, QueryParser
from whoosh.writing import CLEAR

from pyff import merge_strategies
from pyff.constants import ATTRS, ATTRS_INV, NS, config
from pyff.logs import get_log
from pyff.resource import IconHandler
from pyff.samlmd import (
    EntitySet,
    discojson,
    entitiesdescriptor,
    entity_attribute_dict,
    entity_icon_url,
    entity_simple_info,
    entity_simple_summary,
    find_entity,
    find_merge_strategy,
    is_idp,
    is_sp,
    iter_entities,
    object_id,
)
from pyff.utils import (
    LRUProxyDict,
    avg_domain_distance,
    b2u,
    dumptree,
    hash_id,
    hex_digest,
    is_past_ttl,
    is_text,
    load_callable,
    parse_xml,
    redis,
    root,
)

log = get_log(__name__)

DINDEX = ('sha1', 'sha256', 'null')


def make_store_instance(*args, **kwargs):
    new_store = load_callable(config.store_class)
    return new_store(*args, **kwargs)


def make_icon_store_instance(*args, **kwargs):
    new_store = load_callable(config.icon_store_class)
    return new_store(*args, **kwargs)


class Unpickled(object):
    def _pickle(self, data):
        return data

    def _unpickle(self, data):
        return data


class StringSet(Set):
    _pickle = Unpickled._pickle
    _unpickle = Unpickled._unpickle
    _pickle_3 = _pickle
    _unpickle_3 = _unpickle
    _pickle_value = _pickle
    _unpickle_value = _unpickle
    _pickle_key = _pickle
    _unpickle_key = _unpickle


class StringDict(Dict):
    _pickle = Unpickled._pickle
    _unpickle = Unpickled._unpickle
    _pickle_3 = _pickle
    _unpickle_3 = _unpickle
    _pickle_value = _pickle
    _unpickle_value = _unpickle
    _pickle_key = _pickle
    _unpickle_key = _unpickle


class JSONDict(Dict):
    _pickle_key = Unpickled._pickle
    _unpickle_key = Unpickled._unpickle

    def _pickle(self, x):
        return json.dumps(x)

    def _unpickle(self, x):
        return json.loads(b2u(x))

    _pickle_3 = _pickle
    _unpickle_3 = _unpickle
    _pickle_value = _pickle
    _unpickle_value = _unpickle


class XMLDict(Dict):
    _pickle_key = Unpickled._pickle
    _unpickle_key = Unpickled._unpickle

    def _pickle(self, data):
        return dumptree(data)

    def _unpickle(self, pickled_data):
        return root(parse_xml(BytesIO(pickled_data)))

    _pickle_3 = _pickle
    _unpickle_3 = _unpickle
    _pickle_value = _pickle
    _unpickle_value = _unpickle


class IconStore(object):
    def __init__(self):
        pass

    def size(self):
        raise NotImplementedError()

    def lookup(self, uri):
        raise NotImplementedError()

    def update(self, uri, img, info=None):
        raise NotImplementedError()

    def reset(self):
        raise NotImplementedError()

    def is_valid(self, url):
        return True

    def __call__(self, *args, **kwargs):
        watched = kwargs.pop('watched', None)
        scheduler = kwargs.pop('scheduler', None)
        log.debug("about to schedule icon refresh on {} using {}".format(self, scheduler.state))
        if watched is not None and scheduler is not None:
            urls = []
            for r in watched.walk():
                log.debug("looking at {}".format(r.url))
                if r.t is not None:
                    for e in iter_entities(r.t):
                        ico = entity_icon_url(e)
                        if ico is not None and 'url' in ico and not ico['url'].startswith('data:'):
                            urls.append(ico)
            if config.load_icons_async:
                now = datetime.now()
                start = now + timedelta(seconds=20)
                job = scheduler.add_job(
                    IconStore._load_icons,
                    args=[self, urls],
                    id="load_icons",
                    next_run_time=start,
                    name="load_icons",
                    max_instances=1,
                    coalesce=False,
                )
                log.debug(job)
            else:
                self._load_icons(urls)

    def _load_icons(self, urls):
        tbs = []
        for u in [ico['url'] for ico in urls]:
            if not self.is_valid(u):
                tbs.append(u)

        log.debug("fetching {} icons".format(len(tbs)))
        if len(tbs) > 0:
            icon_handler = IconHandler(icon_store=self, name="Icons")
            icon_handler.schedule(tbs)
            try:
                icon_handler.done.acquire()
                icon_handler.done.wait()
            finally:
                icon_handler.done.release()
            icon_handler.fetcher.stop()
            icon_handler.fetcher.join()


class MemoryIconStore(IconStore):
    def __init__(self):
        super().__init__()
        self.icons = {}

    def lookup(self, uri):
        return self.icons.get(uri, None)

    def update(self, uri, img, info=None):
        self.icons[uri] = img

    def reset(self):
        self.icons = {}

    def size(self):
        return len(self.icons)


class RedisIconStore(IconStore):
    def __init__(self, **kwargs):
        super().__init__()
        self._name = kwargs.pop('name', config.store_name)
        self._redis = kwargs.pop('redis', redis())
        clear = bool(kwargs.pop('clear', config.store_clear))
        self._setup()
        if clear:
            self.reset()

    def _setup(self):
        if not self._redis:
            self._redis = redis()  # XXX test cases won't get correctly unpicked because of this
        self.icons = LRUProxyDict(
            JSONDict(key="{}_icons".format(self._name), redis=self._redis, writeback=True), maxsize=config.cache_size
        )

    def lookup(self, uri):
        nfo = self.icons.get(uri, None)
        if nfo is not None and 'data' in nfo:
            return nfo['data']
        return None

    def is_valid(self, url):
        nfo = self.icons.get(url, None)
        if nfo is None or 'last_seen' not in nfo or is_past_ttl(int(nfo['last_seen']), ttl=config.cache_ttl_icons):
            return False
        return True

    def update(self, uri, img, info=None):
        self.icons[uri] = dict(data=img, info=info, last_seen=int(time.time()))

    def __getstate__(self):
        return dict(_name=self._name, _redis=None)

    def __setstate__(self, state):
        state.setdefault('_redis', None)
        self.__dict__.update(state)
        self._setup()

    def reset(self):
        self._redis.delete("{}_icons".format(self._name))

    def size(self):
        return len(self.icons)


class SAMLStoreBase(object):
    def lookup(self, key):
        raise NotImplementedError()

    def __iter__(self):
        for e in self.lookup("entities"):
            log.debug("**** yield entityID=%s" % e.get('entityID'))
            yield e

    def size(self, a=None, v=None):
        raise NotImplementedError()

    def collections(self):
        raise NotImplementedError()

    def update(self, t, tid=None, etag=None, lazy=True):
        raise NotImplementedError()

    def reset(self):
        raise NotImplementedError()

    def entity_ids(self):
        return set(e.get('entityID') for e in self.lookup('entities'))

    def _select(self, member=None):
        if member is None:
            member = "entities"

        if is_text(member):
            if '!' in member:
                (src, xp) = member.split("!")
                if len(src) == 0:
                    src = None
                return self.select(src, xp=xp)

        log.debug("calling store lookup %s" % member)
        return self.lookup(member)

    def __call__(self, *args, **kwargs):
        watched = kwargs.pop('watched', None)
        scheduler = kwargs.pop('scheduler', None)
        if watched is not None and scheduler is not None:
            for r in watched.walk():
                if r.t is None and len(r.children) > 0:
                    r.t = entitiesdescriptor(list(filter(lambda c: c is not None, [c.t for c in r.children])), name=r.name, validate=True, filter_invalid=True)
                if r.t is not None:
                    self.update(r.t, tid=r.name, etag=r.etag)
                else:
                    log.debug(f'Nothing to update for resource {r.name} with {len(r.children)} children')

    def select(self, member, xp=None):
        """
        Select a set of metadata elements and return an EntityDescriptor with the result of the select.

        :param member: A selector (cf below)
        :type member: basestring
        :param xp: An optional xpath filter
        :type xp: basestring
        :return: An interable of EntityDescriptor elements
        :rtype: etree.Element


        **Selector Syntax**

            - selector "+" selector
            - [sourceID] "!" xpath
            - attribute=value or {attribute}value
            - entityID
            - source (typically @Name from an EntitiesDescriptor set but could also be an alias)

        The first form results in the intersection of the results of doing a lookup on the selectors. The second form
        results in the EntityDescriptor elements from the source (defaults to all EntityDescriptors) that match the
        xpath expression. The attribute-value forms resuls in the EntityDescriptors that contain the specified entity
        attribute pair. If non of these forms apply, the lookup is done using either source ID (normally @Name from
        the EntitiesDescriptor) or the entityID of single EntityDescriptors. If member is a URI but isn't part of
        the metadata repository then it is fetched an treated as a list of (one per line) of selectors. If all else
        fails an empty list is returned.
        """
        l = self._select(member)
        if hasattr(l, 'tag'):
            l = [l]
        elif hasattr(l, '__iter__'):
            l = list(l)

        if xp is None:
            return l
        else:
            log.debug("filtering %d entities using xpath %s" % (len(l), xp))
            t = entitiesdescriptor(l, 'dummy', lookup_fn=self.lookup)
            if t is None:
                return []
            l = root(t).xpath(xp, namespaces=NS, smart_strings=False)
            log.debug("got %d entities after filtering" % len(l))
            return l

    def merge(self, t, nt, strategy=merge_strategies.replace_existing, strategy_name=None):
        """
        :param t: The EntitiesDescriptor element to merge *into*
        :param nt:  The EntitiesDescriptor element to merge *from*
        :param strategy: A callable implementing the merge strategy pattern
        :param strategy_name: The name of a strategy to import. Overrides the callable if present.
        :return:

        Two EntitiesDescriptor elements are merged - the second into the first. For each element
        in the second collection that is present (using the @entityID attribute as key) in the
        first the strategy callable is called with the old and new EntityDescriptor elements
        as parameters. The strategy callable thus must implement the following pattern:

        :old_e: The EntityDescriptor from t
        :e: The EntityDescriptor from nt
        :return: A merged EntityDescriptor element

        Before each call to strategy old_e is removed from the MDRepository index and after
        merge the resultant EntityDescriptor is added to the index before it is used to
        replace old_e in t.
        """
        if strategy_name is not None:
            strategy = find_merge_strategy(strategy_name)

        for e in iter_entities(nt):
            entity_id = e.get("entityID")
            # we assume ddup:ed tree
            old_e = find_entity(t, entity_id)
            new = strategy(old_e, e)
            if new is not None:
                self.update(new)

    def search(self, query=None, path=None, entity_filter=None, related=None):
        """
        :param query: A string to search for.
        :param path: The repository collection (@Name) to search in - None for search in all collections
        :param entity_filter: An optional lookup expression used to filter the entries before search is done.
        :param related: an optional '+'-separated list of related domain names for prioritizing search results

        Returns a list of dict's for each EntityDescriptor present in the metadata store such
        that any of the DisplayName, ServiceName, OrganizationName or OrganizationDisplayName
        elements match the query (as in contains the query as a substring).

        The dict in the list contains three items:

        :title: A displayable string, useful as a UI label
        :value: The entityID of the EntityDescriptor
        :id: A sha1-ID of the entityID - on the form {sha1}<sha1-hash-of-entityID>
        """

        match_query = bool(len(query) > 0)

        if isinstance(query, six.string_types):
            query = [query.lower()]

        def _strings(elt):
            lst = []
            for attr in [
                './/{%s}UIInfo/{%s}DisplayName' % (NS['mdui'], NS['mdui']),
                '{%s}ServiceName' % NS['md'],
                '{%s}OrganizationDisplayName' % NS['md'],
                '{%s}OrganizationName' % NS['md'],
                './/{%s}UIInfo/{%s}Keywords' % (NS['mdui'], NS['mdui']),
                '{%s}Scope' % NS['shibmd'],
            ]:
                lst.extend([s.text for s in elt.iterfind(attr)])
            lst.append(elt.get('entityID'))
            return [item for item in lst if item is not None]

        def _ip_networks(elt):
            return [ipaddress.ip_network(x.text) for x in elt.iter('{%s}IPHint' % NS['mdui'])]

        def _match(qq, elt):
            for q in qq:
                q = q.strip()
                if ':' in q or '.' in q:
                    try:
                        nets = _ip_networks(elt)
                        for net in nets:
                            if ipaddress.ip_address(q) in net:
                                return net
                    except ValueError:
                        pass

                if q is not None and len(q) > 0:
                    tokens = _strings(elt)
                    for tstr in tokens:
                        if q in tstr.lower():
                            return tstr

            return None

        f = []
        if path is not None and path not in f:
            f.append(path)
        if entity_filter is not None and entity_filter not in f:
            f.append(entity_filter)
        mexpr = None
        if f:
            mexpr = "+".join(f)

        log.debug("match using '%s'" % mexpr)
        res = []
        for e in self.lookup(mexpr):
            d = None
            if match_query:
                m = _match(query, e)
                if m is not None:
                    d = entity_simple_summary(e)
                    ll = d['title'].lower()
                    d['matched'] = m
            else:
                d = entity_simple_summary(e)

            if d is not None:
                if related is not None:
                    d['ddist'] = avg_domain_distance(related, d['domains'])
                else:
                    d['ddist'] = 0

                res.append(d)

        res.sort(key=operator.itemgetter('title'))
        res.sort(key=operator.itemgetter('ddist'), reverse=True)

        return res


class EmptyStore(SAMLStoreBase):
    def lookup(self, key):
        return list()

    def __init__(self, *args, **kwargs):
        pass

    def update(self, *args, **kwargs):
        return 0

    def size(self, **kwargs):
        return 0

    def collections(self):
        return []

    def reset(self):
        pass

    def entity_ids(self):
        return set()

    def select(self, **kwargs):
        return list()

    def search(self, **kwargs):
        return list()

    def merge(self, *args, **kwargs):
        return list()


class RedisWhooshStore(SAMLStoreBase):  # TODO: This needs a gc mechanism for keys (uuids)
    def json_dict(self, name):
        return LRUProxyDict(
            JSONDict(key='{}_{}'.format(self._name, name), redis=self._redis, writeback=True), maxsize=config.cache_size
        )

    def xml_dict(self, name):
        return LRUProxyDict(
            XMLDict(key='{}_{}'.format(self._name, name), redis=self._redis, writeback=True), maxsize=config.cache_size
        )

    def __init__(self, *args, **kwargs):
        self._dir = kwargs.pop('directory', '.whoosh')
        clear = bool(kwargs.pop('clear', config.store_clear))
        self._name = kwargs.pop('name', config.store_name)
        self._redis = kwargs.pop('redis', redis())
        if clear:
            shutil.rmtree(self._dir)
        now = datetime.now()
        self._last_index_time = now
        self._last_modified = now
        self._setup()
        if clear:
            self.reset()

    def _setup(self):
        self._redis = getattr(self, '_redis', None)
        if not self._redis:
            self._redis = redis()  # XXX test cases won't get correctly unpicked because of this
        self.schema = Schema(content=NGRAMWORDS(stored=False))
        self.schema.add("object_id", ID(stored=True, unique=True))
        self.schema.add("entity_id", ID(stored=True, unique=True))
        self.schema.add('sha1', ID(stored=True, unique=True))
        for a in list(ATTRS.keys()):
            self.schema.add(a, KEYWORD())
        self.objects = self.xml_dict('objects')
        self.parts = self.json_dict('parts')
        self.storage = FileStorage(os.path.join(self._dir, self._name))
        try:
            self.index = self.storage.open_index(schema=self.schema)
        except BaseException as ex:
            log.warning(ex)
            self.storage.create()
            self.index = self.storage.create_index(self.schema)
            self._reindex()

    def __getstate__(self):
        state = dict()
        for p in ('_dir', '_name', '_last_index_time', '_last_modified'):
            state[p] = getattr(self, p)
        return state

    def __setstate__(self, state):
        self.__dict__.update(state)
        self._setup()

    def __call__(self, *args, **kwargs):
        watched = kwargs.pop('watched', None)
        scheduler = kwargs.pop('scheduler', None)
        if watched is not None and scheduler is not None:
            super(RedisWhooshStore, self).__call__(watched=watched, scheduler=scheduler)
            log.debug("indexing using {}".format(scheduler))
            if scheduler is not None:  # and self._last_modified > self._last_index_time and :
                scheduler.add_job(
                    RedisWhooshStore._reindex,
                    args=[self],
                    max_instances=1,
                    coalesce=True,
                    misfire_grace_time=2 * config.update_frequency,
                )

    def _reindex(self):
        log.debug("indexing the store...")
        self._last_index_time = datetime.now()
        seen = set()
        refs = set([b2u(s) for s in self.objects.keys()])
        parts = self.parts.values()
        for ref in refs:
            for part in parts:
                if ref in part['items']:
                    seen.add(ref)

        ix = self.storage.open_index()
        lock = ix.lock("reindex")
        try:
            log.debug("waiting for index lock")
            lock.acquire(True)
            log.debug("got index lock")
            with ix.writer() as writer:
                for ref in refs:
                    if ref not in seen:
                        log.debug("removing unseen ref {}".format(ref))
                        del self.objects[ref]
                        del self.parts[ref]

                log.debug("updating index")
                for e in self.objects.values():
                    info = self._index_prep(entity_simple_info(e))
                    ref = object_id(e)
                    writer.add_document(object_id=ref, **info)

                writer.mergetype = CLEAR
        finally:
            try:
                log.debug("releasing index lock")
                lock.release()
            except ThreadError as ex:
                pass

    def dump(self):
        ix = self.storage.open_index()
        from whoosh.query import Every

        with ix.searcher() as searcher:
            for result in ix.searcher().search(Every('object_id')):
                print(result)

    def _index_prep(self, info):
        res = dict()
        if 'entity_attributes' in info:
            for a, v in list(info.pop('entity_attributes').items()):
                info[a] = v

        content = " ".join(
            filter(
                lambda x: x is not None,
                [info.get(x, '') for x in ('service_name', 'title', 'domain', 'keywords', 'scopes')],
            )
        )
        res['content'] = content.strip()
        for a, v in info.items():
            k = a
            if a in ATTRS_INV:
                k = ATTRS_INV[a]

            if k in self.schema.names():
                if type(v) in (list, tuple):
                    res[k] = " ".join([vv.lower() for vv in v])
                elif type(v) in six.string_types:
                    res[k] = info[a].lower()
        res['sha1'] = hash_id(info['entity_id'], prefix=False)
        return res

    def update(self, t, tid=None, etag=None, lazy=True):
        relt = root(t)
        assert relt is not None

        if relt.tag == "{%s}EntityDescriptor" % NS['md']:
            ref = object_id(relt)
            parts = None
            if ref in self.parts:
                parts = self.parts[ref]
            if etag is not None and (parts is None or parts.get('etag', None) != etag):
                self.parts[ref] = {'id': relt.get('entityID'), 'etag': etag, 'count': 1, 'items': [ref]}
                self.objects[ref] = relt
                self._last_modified = datetime.now()
        elif relt.tag == "{%s}EntitiesDescriptor" % NS['md']:
            if tid is None:
                tid = relt.get('Name')
            if etag is None:
                etag = hex_digest(dumptree(t, pretty_print=False), 'sha256')
            parts = None
            if tid in self.parts:
                parts = self.parts[tid]
            if parts is None or parts.get('etag', None) != etag:
                items = set()
                for e in iter_entities(t):
                    ref = object_id(e)
                    items.add(ref)
                    self.objects[ref] = e
                self.parts[tid] = {'id': tid, 'count': len(items), 'etag': etag, 'items': list(items)}
                self._last_modified = datetime.now()

        if not lazy:
            self._reindex()

    @ttl_cache(ttl=config.cache_ttl, maxsize=config.cache_size)
    def collections(self):
        return [b2u(ref) for ref in self.parts.keys()]

    def reset(self):
        for k in ('{}_{}'.format(self._name, 'parts'), '{}_{}'.format(self._name, 'objects')):
            self._redis.delete('{}_{}'.format(self._name, 'parts'))
            self._redis.delete('{}_{}'.format(self._name, 'objects'))

    def size(self, a=None, v=None):
        if a is None:
            return len(self.objects.keys())
        elif a is not None and v is None:
            return len(self.attribute(a))
        else:
            return len(self.lookup("{!s}={!s}".format(a, v)))

    def _attributes(self):
        ix = self.storage.open_index()
        with ix.reader() as reader:
            for n in reader.indexed_field_names():
                if n in ATTRS:
                    yield b2u(ATTRS[n])

    def attributes(self):
        return b2u(list(self._attributes()))

    def attribute(self, a):
        if a in ATTRS_INV:
            n = ATTRS_INV[a]
            ix = self.storage.open_index()
            with ix.searcher() as searcher:
                return b2u(list(searcher.lexicon(n)))
        else:
            return []

    def _prep_key(self, key):
        # import pdb; pdb.set_trace()
        key = key.strip('+')
        key = key.replace('+', ' AND ')
        key = key.replace('-', ' AND NOT ')
        for uri, a in list(ATTRS_INV.items()):
            key = key.replace(uri, a)
        key = " {!s} ".format(key)
        key = re.sub("([^=]+)=(\S+)", "\\1:\\2", key)
        key = re.sub("{([^}]+)}(\S+)", "\\1:\\2", key)
        key = key.strip()

        return key

    def _entities(self):
        lst = set()
        for ref_data in self.parts.values():
            for ref in ref_data['items']:
                e = self.objects.get(ref, None)
                if e is not None:
                    lst.add(e)

        return b2u(list(lst))

    @ttl_cache(ttl=config.cache_ttl, maxsize=config.cache_size)
    def lookup(self, key):
        if key == 'entities' or key is None:
            return self._entities()

        bkey = six.b(key)
        if bkey in self.objects:
            return [self.objects.get(bkey)]

        if bkey in self.parts:
            res = []
            part = self.parts.get(bkey)
            for item in part['items']:
                res.extend(self.lookup(item))
            return res

        key = self._prep_key(key)
        qp = QueryParser("object_id", schema=self.schema)
        q = qp.parse(key)
        lst = set()
        with self.index.searcher() as searcher:
            results = searcher.search(q, limit=None)
            for result in results:
                e = self.objects.get(result['object_id'], None)
                if e is not None:
                    lst.add(e)

        return b2u(list(lst))

    @ttl_cache(ttl=config.cache_ttl, maxsize=config.cache_size)
    def search(self, query=None, path=None, entity_filter=None, related=None):
        if entity_filter:
            query = "{!s} AND {!s}".format(query, entity_filter)
        query = self._prep_key(query)
        qp = MultifieldParser(['content', 'domain'], schema=self.schema)
        q = qp.parse(query)
        lst = set()
        with self.index.searcher() as searcher:
            results = searcher.search(q, limit=None)
            log.debug(results)
            for result in results:
                lst.add(result['object_id'])

        res = list()
        for ref in lst:
            e = self.objects.get(ref, None)
            if e is not None:
                res.append(discojson(e))
        return res


class MemoryStore(SAMLStoreBase):
    def __init__(self, *args, **kwargs):
        self.md = dict()
        self.index = dict()
        self.entities = dict()

        for hn in DINDEX:
            self.index.setdefault(hn, {})
        self.index.setdefault('attr', {})

    def __str__(self):
        return repr(self.index)

    def size(self, a=None, v=None):
        if a is None:
            return len(self.entities)
        elif a is not None and v is None:
            return len(list(self.index.setdefault('attr', {}).setdefault(a, {}).keys()))
        else:
            return len(self.index.setdefault('attr', {}).setdefault(a, {}).get(v, []))

    def attributes(self):
        return list(self.index.setdefault('attr', {}).keys())

    def attribute(self, a):
        return list(self.index.setdefault('attr', {}).setdefault(a, {}).keys())

    def _modify(self, entity, modifier):
        def _m(idx, vv):
            getattr(idx.setdefault(vv, EntitySet()), modifier)(entity)

        for hn in DINDEX:
            _m(self.index[hn], hash_id(entity, hn, False))

        attr_idx = self.index.setdefault('attr', {})
        for attr, values in list(entity_attribute_dict(entity).items()):
            vidx = attr_idx.setdefault(attr, {})
            for v in values:
                _m(vidx, v)

        vidx = attr_idx.setdefault(ATTRS['role'], {})
        if is_idp(entity):
            _m(vidx, "idp")
        if is_sp(entity):
            _m(vidx, "sp")

    def _index(self, entity):
        return self._modify(entity, "add")

    def _unindex(self, entity):
        return self._modify(entity, "discard")

    def _get_index(self, a, v):
        if a in DINDEX:
            return self.index[a].get(v, [])
        else:
            idx = self.index['attr'].setdefault(a, {})
            entities = idx.get(v, None)
            if entities is not None:
                return entities
            else:
                m = re.compile(v)
                entities = []
                for value, ents in list(idx.items()):
                    if m.match(value):
                        entities.extend(ents)
                return entities

    def reset(self):
        self.__init__()

    def collections(self):
        return list(self.md.keys())

    def update(self, t, tid=None, etag=None, lazy=True):
        relt = root(t)
        assert relt is not None
        if relt.tag == "{%s}EntityDescriptor" % NS['md']:
            self._unindex(relt)
            self._index(relt)
            self.entities[relt.get('entityID')] = relt  # TODO: merge?
            if tid is not None:
                self.md[tid] = [relt.get('entityID')]
        elif relt.tag == "{%s}EntitiesDescriptor" % NS['md']:
            if tid is None:
                tid = relt.get('Name')
            lst = []
            for e in iter_entities(t):
                self.update(e)
                lst.append(e.get('entityID'))
            self.md[tid] = lst

    def lookup(self, key):
        return self._lookup(key)

    def _lookup(self, key):
        if key == 'entities' or key is None:
            return list(self.entities.values())

        if key in self.entities:
            return [self.entities[key]]

        if '+' in key:
            key = key.strip('+')
            # log.debug("lookup intersection of '%s'" % ' and '.join(key.split('+')))
            hits = None
            for f in key.split("+"):
                f = f.strip()
                if hits is None:
                    hits = set(self._lookup(f))
                else:
                    other = self._lookup(f)
                    hits.intersection_update(other)

                if not hits:
                    # log.debug("empty intersection")
                    return []

            if hits is not None and hits:
                return list(hits)
            else:
                return []

        m = re.match("^(.+)=(.+)$", key)
        if m:
            return self._lookup("{%s}%s" % (m.group(1), str(m.group(2)).rstrip("/")))

        m = re.match("^{(.+)}(.+)$", key)
        if m:
            res = set()
            for v in str(m.group(2)).rstrip("/").split(';'):
                # log.debug("... adding %s=%s" % (m.group(1),v))
                res.update(self._get_index(m.group(1), v))
            return list(res)

        if key in self.md:
            log.debug("entities list %s: %d" % (key, len(self.md[key])))
            lst = []
            for entityID in self.md[key]:
                lst.extend(self.lookup(entityID))
            log.debug("returning {} entities".format(len(lst)))
            return lst

        return []