lazy.py 5.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173
  1. from threading import RLock
  2. try:
  3. from collections.abc import Mapping as DictMixin
  4. except ImportError: # Python < 3.3
  5. try:
  6. from UserDict import DictMixin # Python 2
  7. except ImportError: # Python 3.0-3.3
  8. from collections import Mapping as DictMixin
  9. # With lazy loading, we might end up with multiple threads triggering
  10. # it at the same time. We need a lock.
  11. _fill_lock = RLock()
  12. class LazyDict(DictMixin):
  13. """Dictionary populated on first use."""
  14. data = None
  15. def __getitem__(self, key):
  16. if self.data is None:
  17. _fill_lock.acquire()
  18. try:
  19. if self.data is None:
  20. self._fill()
  21. finally:
  22. _fill_lock.release()
  23. return self.data[key.upper()]
  24. def __contains__(self, key):
  25. if self.data is None:
  26. _fill_lock.acquire()
  27. try:
  28. if self.data is None:
  29. self._fill()
  30. finally:
  31. _fill_lock.release()
  32. return key in self.data
  33. def __iter__(self):
  34. if self.data is None:
  35. _fill_lock.acquire()
  36. try:
  37. if self.data is None:
  38. self._fill()
  39. finally:
  40. _fill_lock.release()
  41. return iter(self.data)
  42. def __len__(self):
  43. if self.data is None:
  44. _fill_lock.acquire()
  45. try:
  46. if self.data is None:
  47. self._fill()
  48. finally:
  49. _fill_lock.release()
  50. return len(self.data)
  51. def keys(self):
  52. if self.data is None:
  53. _fill_lock.acquire()
  54. try:
  55. if self.data is None:
  56. self._fill()
  57. finally:
  58. _fill_lock.release()
  59. return self.data.keys()
  60. class LazyList(list):
  61. """List populated on first use."""
  62. _props = [
  63. '__str__', '__repr__', '__unicode__',
  64. '__hash__', '__sizeof__', '__cmp__',
  65. '__lt__', '__le__', '__eq__', '__ne__', '__gt__', '__ge__',
  66. 'append', 'count', 'index', 'extend', 'insert', 'pop', 'remove',
  67. 'reverse', 'sort', '__add__', '__radd__', '__iadd__', '__mul__',
  68. '__rmul__', '__imul__', '__contains__', '__len__', '__nonzero__',
  69. '__getitem__', '__setitem__', '__delitem__', '__iter__',
  70. '__reversed__', '__getslice__', '__setslice__', '__delslice__']
  71. def __new__(cls, fill_iter=None):
  72. if fill_iter is None:
  73. return list()
  74. # We need a new class as we will be dynamically messing with its
  75. # methods.
  76. class LazyList(list):
  77. pass
  78. fill_iter = [fill_iter]
  79. def lazy(name):
  80. def _lazy(self, *args, **kw):
  81. _fill_lock.acquire()
  82. try:
  83. if len(fill_iter) > 0:
  84. list.extend(self, fill_iter.pop())
  85. for method_name in cls._props:
  86. delattr(LazyList, method_name)
  87. finally:
  88. _fill_lock.release()
  89. return getattr(list, name)(self, *args, **kw)
  90. return _lazy
  91. for name in cls._props:
  92. setattr(LazyList, name, lazy(name))
  93. new_list = LazyList()
  94. return new_list
  95. # Not all versions of Python declare the same magic methods.
  96. # Filter out properties that don't exist in this version of Python
  97. # from the list.
  98. LazyList._props = [prop for prop in LazyList._props if hasattr(list, prop)]
  99. class LazySet(set):
  100. """Set populated on first use."""
  101. _props = (
  102. '__str__', '__repr__', '__unicode__',
  103. '__hash__', '__sizeof__', '__cmp__',
  104. '__lt__', '__le__', '__eq__', '__ne__', '__gt__', '__ge__',
  105. '__contains__', '__len__', '__nonzero__',
  106. '__getitem__', '__setitem__', '__delitem__', '__iter__',
  107. '__sub__', '__and__', '__xor__', '__or__',
  108. '__rsub__', '__rand__', '__rxor__', '__ror__',
  109. '__isub__', '__iand__', '__ixor__', '__ior__',
  110. 'add', 'clear', 'copy', 'difference', 'difference_update',
  111. 'discard', 'intersection', 'intersection_update', 'isdisjoint',
  112. 'issubset', 'issuperset', 'pop', 'remove',
  113. 'symmetric_difference', 'symmetric_difference_update',
  114. 'union', 'update')
  115. def __new__(cls, fill_iter=None):
  116. if fill_iter is None:
  117. return set()
  118. class LazySet(set):
  119. pass
  120. fill_iter = [fill_iter]
  121. def lazy(name):
  122. def _lazy(self, *args, **kw):
  123. _fill_lock.acquire()
  124. try:
  125. if len(fill_iter) > 0:
  126. for i in fill_iter.pop():
  127. set.add(self, i)
  128. for method_name in cls._props:
  129. delattr(LazySet, method_name)
  130. finally:
  131. _fill_lock.release()
  132. return getattr(set, name)(self, *args, **kw)
  133. return _lazy
  134. for name in cls._props:
  135. setattr(LazySet, name, lazy(name))
  136. new_set = LazySet()
  137. return new_set
  138. # Not all versions of Python declare the same magic methods.
  139. # Filter out properties that don't exist in this version of Python
  140. # from the list.
  141. LazySet._props = [prop for prop in LazySet._props if hasattr(set, prop)]