Source code for viscid.bucket

#!/usr/bin/env python

from __future__ import print_function
import sys

# from viscid.vutil import tree_prefix
from viscid.compat import OrderedDict
from viscid import logger

[docs]class Bucket(object): """ This is basically a glorified dict It's a convenient dict-like object if you want lots of keys for a given value. NOTE: You can add non-hashable items, but this is poorly tested. When adding / removing non-hashable items (items, not handles) the comparison is done using the object's id. This is fundamentally different than using an object's __hash__, but it should be fairly transparent. """ _ordered = False _ref_count = None # keys are hashable items, values are # of times item was added _hash_lookup = None # keys are hashable items, values are actual items _handles = None # keys are hashable items, values are list of handles _items = None # keys are handles, values are actual items # if index handle, set_item adds this number as a handle and increments it # this is useful for hiding loads that are not user initiated, such as # an xdmf file loading an h5 file under the covers _int_counter = None def __init__(self, ordered=False): self._ordered = ordered self._set_empty_dicts() self._int_counter = 0 def _set_empty_dicts(self): if self._ordered: self._ref_count = OrderedDict() self._hash_lookup = OrderedDict() self._handles = OrderedDict() self._items = OrderedDict() else: self._ref_count = {} self._hash_lookup = {} self._handles = {} self._items = {} @staticmethod def _make_hashable(item): try: hash(item) return item except TypeError: return "<{0} @ {1}>".format(type(item), hex(id(item)))
[docs] def items(self): for hashable_item, item in self._hash_lookup.items(): yield self._handles[hashable_item], item
[docs] def keys(self): return self._handles.values()
[docs] def values(self): return self._hash_lookup.values()
[docs] def set_item(self, handles, item, index_handle=True, _add_ref=False): """ if index_handle is true then the index of item will be included as a handle making the bucket indexable like a list """ # found = False if handles is None: handles = [] if not isinstance(handles, list): raise TypeError("handle must by of list type") # make sure we have a hashable "item" for doing reverse # lookups of handles using an item hashable_item = self._make_hashable(item) if hashable_item not in self._hash_lookup: if index_handle: handles += [self._int_counter] self._int_counter += 1 handles_added = [] for h in handles: # check if we're stealing a handle from another item try: hash(h) except TypeError: logger.error("A bucket says handle '{0}' is not hashable, " "ignoring it".format(h)) continue if (h in self._items) and (item is self._items[h]): continue elif h in self._items: logger.error("The handle '{0}' is being hijacked! Memory leak " "could ensue.".format(h)) # romove handle from old item, since this check is here, # there sholdn't be 2 items with the same handle in the # items dict old_item = self._items[h] old_hashable_item = self._make_hashable(old_item) self._handles[old_hashable_item].remove(h) if len(self._handles[old_hashable_item]) == 0: self.remove_item(old_item) self._items[h] = item handles_added.append(h) try: self._handles[hashable_item] += handles_added if _add_ref: self._ref_count[hashable_item] += 1 except KeyError: if len(handles_added) == 0: logger.error("No valid handles given, item '{0}' not added to " "bucket".format(hashable_item)) else: self._handles[hashable_item] = handles_added self._hash_lookup[hashable_item] = item self._ref_count[hashable_item] = 1 return None
def _remove_item(self, item): """remove item no matter what You may want to use remove_ , raises ValueError if item is not found """ hashable_item = self._make_hashable(item) handles = self._handles[hashable_item] for h in handles: del self._items[h] del self._hash_lookup[hashable_item] del self._handles[hashable_item] del self._ref_count[hashable_item] def _remove_item_by_handle(self, handle): self._remove_item(self._items[handle])
[docs] def remove_item(self, item): self._remove_item(item)
[docs] def remove_item_by_handle(self, handle): """ remove item by handle, raises KeyError if handle is not found """ self.remove_item(self._items[handle])
[docs] def remove_reference(self, item, _ref_count=1): hashable_item = self._make_hashable(item) try: self._ref_count[hashable_item] -= _ref_count except KeyError: item = self[item] hashable_item = self._make_hashable(item) if _ref_count: self._ref_count[hashable_item] -= _ref_count else: self._ref_count[hashable_item] = 0 # FIXME: unload_all_files breaks this assert check... probably a bug # assert self._ref_count[hashable_item] >= 0, \ # "problem with bucket ref counting {0}".format(hashable_item) if self._ref_count[hashable_item] <= 0: self._remove_item(item)
[docs] def remove_all_items(self): """ unload all items """ self._set_empty_dicts()
[docs] def items_as_list(self): return list(self._hash_lookup.values())
[docs] def get_primary_handles(self): """Return a list of the first handles for all items""" return [handles[0] for handles in self._handles.values()]
[docs] def handle_string(self, prefix=""): """ return string representation of handles and items """ # this is inefficient, but probably doesn't matter s = "" for item, handles in self._handles.items(): hands = [repr(h) for h in handles] s += "{0}handles: {1}\n".format(prefix, ", ".join(hands)) s += "{0} item: {1}\n".format(prefix, str(item)) return s
[docs] def print_tree(self, prefix=""): print(self.handle_string(prefix=prefix), end='')
def __getitem__(self, handle): return self._items[handle] def __setitem__(self, key, value): if isinstance(key, (list, tuple)): key = list(key) elif key is not None: key = [key] self.set_item(key, value) def __delitem__(self, handle): try: self.remove_item_by_handle(handle) except (KeyError, TypeError): # maybe we are asking to remove an item explicitly self.remove_item(handle) def __iter__(self): return self.values().__iter__()
[docs] def contains_item(self, item): hashable_item = self._make_hashable(item) return hashable_item in self._handles
[docs] def contains_handle(self, handle): try: return handle in self._items except TypeError: return False
def __contains__(self, handle): return self.contains_handle(handle) or self.contains_item(handle) def __len__(self): return len(self._hash_lookup) def __str__(self): return self.handle_string()
def _main(): import os import viscid sample_dir = os.path.join(viscid.sample_dir, "local_0001") sample_prefix = sample_dir + "local_0001" fnpy0 = sample_prefix + ".py_0.xdmf" fn3dfa = sample_prefix + ".3df.xdmf" fn3df = sample_prefix + ".3df.004200.xdmf" fniof = sample_prefix + ".iof.004200.xdmf" fnasc = sample_dir + "test.asc" fnascts = sample_dir + "test_time.asc" # fm = get_file_bucket() # print("load {0}".format(fnasc)) # fasc = fm.add(fnasc) # print("load {0}".format(fnascts)) # fasc_time = fm.add(fnascts) # print("load 3df.xdmf") # d3da = fm.load(fn3dfa) # print("load 3df.004200.xdmf") # f3d = load_file(fn3df) # print("load py_0.xdmf") # fpy0 = load_file(fnpy0) # print("load test.xdmf") # ftest = fm.load('../sample/test.xdmf') viscid.interact() print("done") # a=re.findall('(\S+)\s*=\s*(?:[\'"](.*?)[\'"]|(.*?))(?:\s|$)', # 'key="value" delim=" " socrates k=v', flags=0); print(a) return 0 if __name__ == '__main__': sys.exit(_main()) ## ## EOF ##