Rev 134: More attributes, almost ready to replace the ObjManager.objs dict. in http://bazaar.launchpad.net/~jameinel/meliae/mem-object-collection
John Arbash Meinel
john at arbash-meinel.com
Mon Dec 28 05:06:32 GMT 2009
At http://bazaar.launchpad.net/~jameinel/meliae/mem-object-collection
------------------------------------------------------------
revno: 134
revision-id: john at arbash-meinel.com-20091228050617-h1znhvissa2tkzyp
parent: john at arbash-meinel.com-20091228044038-bkwaqjf1p9feehvv
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: mem-object-collection
timestamp: Sun 2009-12-27 23:06:17 -0600
message:
More attributes, almost ready to replace the ObjManager.objs dict.
-------------- next part --------------
=== modified file 'meliae/_loader.pyx'
--- a/meliae/_loader.pyx 2009-12-28 04:40:38 +0000
+++ b/meliae/_loader.pyx 2009-12-28 05:06:17 +0000
@@ -138,8 +138,8 @@
# Consider making this unsigned long
long size
RefList *ref_list
- # TODO: Scheduled for removal
- int length
+ # Removed for now, since it hasn't proven useful
+ # int length
PyObject *value
PyObject *name
RefList *referrer_list
@@ -216,6 +216,16 @@
Py_DECREF(self._obj.value)
self._obj.value = new_val
+ property total_size:
+ """Mean to hold the size of this plus size of all referenced objects."""
+ def __get__(self):
+ self._ensure_obj()
+ return self._obj.total_size
+
+ def __set__(self, value):
+ self._ensure_obj()
+ self._obj.total_size = value
+
def __len__(self):
self._ensure_obj()
if self._obj.ref_list == NULL:
@@ -280,6 +290,9 @@
memset(self._table, 0, sizeof(_MemObject*)*1024)
self._proxies = weakref.WeakValueDictionary()
+ def __len__(self):
+ return self._active
+
cdef _MemObject** _lookup(self, address) except NULL:
cdef long the_hash
cdef size_t i, n_lookup
@@ -355,6 +368,17 @@
return False
return True
+ cdef _MemObjectProxy _proxy_for(self, address, _MemObject *val):
+ cdef _MemObjectProxy proxy
+
+ if address in self._proxies:
+ proxy = self._proxies[address]
+ else:
+ proxy = _MemObjectProxy(self)
+ proxy._obj = val
+ self._proxies[address] = proxy
+ return proxy
+
def __getitem__(self, at):
cdef _MemObject **slot
cdef _MemObjectProxy proxy
@@ -370,14 +394,15 @@
if slot[0] == NULL or slot[0] == _dummy:
raise KeyError('address %s not present' % (at,))
if proxy is None:
- if address in self._proxies:
- proxy = self._proxies[address]
- else:
- proxy = _MemObjectProxy(self)
- proxy._obj = slot[0]
- self._proxies[address] = proxy
+ proxy = self._proxy_for(address, slot[0])
return proxy
+ def get(self, at, default=None):
+ try:
+ return self[at]
+ except KeyError:
+ return default
+
def __delitem__(self, at):
cdef _MemObject **slot
cdef _MemObjectProxy proxy
@@ -395,6 +420,7 @@
proxy._obj = NULL
self._clear_slot(slot)
slot[0] = _dummy
+ self._active -= 1
# TODO: Shrink
#def __setitem__(self, address, value):
@@ -506,10 +532,10 @@
new_entry.size = size
new_entry.ref_list = _list_to_ref_list(ref_list)
# TODO: Scheduled for removal
- if length is None:
- new_entry.length = -1
- else:
- new_entry.length = length
+ # if length is None:
+ # new_entry.length = -1
+ # else:
+ # new_entry.length = length
new_entry.value = <PyObject *>value
Py_INCREF(new_entry.value)
new_entry.name = <PyObject *>name
@@ -533,6 +559,41 @@
PyMem_Free(self._table)
self._table = NULL
+ def iteritems(self):
+ """Iterate over (key, value) tuples."""
+ cdef long i
+ cdef _MemObject *cur
+ cdef _MemObjectProxy proxy
+
+ values = []
+ for i from 0 <= i < self._table_mask:
+ cur = self._table[i]
+ if cur == NULL or cur == _dummy:
+ continue
+ else:
+ address = <object>cur.address
+ proxy = self._proxy_for(address, cur)
+ values.append((address, proxy))
+ return values
+
+ def itervalues(self):
+ """Return an iterable of values stored in this map."""
+ # This returns a list, but that is 'close enough' for what we need
+ cdef long i
+ cdef _MemObject *cur
+ cdef _MemObjectProxy proxy
+
+ values = []
+ for i from 0 <= i < self._table_mask:
+ cur = self._table[i]
+ if cur == NULL or cur == _dummy:
+ continue
+ else:
+ proxy = self._proxy_for(<object>cur.address, cur)
+ values.append(proxy)
+ return values
+
+
cdef class MemObject:
"""This defines the information we know about the objects.
=== modified file 'meliae/loader.py'
--- a/meliae/loader.py 2009-12-28 04:13:37 +0000
+++ b/meliae/loader.py 2009-12-28 05:06:17 +0000
@@ -548,16 +548,24 @@
% (line_num, len(objs), mb_read, input_mb, tdelta))
+def _load_moc(source, using_json, show_prog, input_size):
+ objs = _loader.MemObjectCollection()
+ for memobj in iter_objs(source, using_json, show_prog, input_size, objs,
+ factory=objs.add):
+ # objs.add automatically adds the object as it is created
+ pass
+ return ObjManager(objs, show_progress=show_prog)
+
+
def _load(source, using_json, show_prog, input_size):
- #objs = _loader.MemObjectCollection()
objs = {}
- for memobj in iter_objs(source, using_json, show_prog, input_size, objs,
- ):#factory=objs.add):
- # objs.add automatically adds the object as it is created
+ for memobj in iter_objs(source, using_json, show_prog, input_size, objs):
objs[memobj.address] = memobj
# _fill_total_size(objs)
return ObjManager(objs, show_progress=show_prog)
+#_load = _load_moc
+
def remove_expensive_references(source, total_objs=0, show_progress=False):
"""Filter out references that are mere houskeeping links.
=== modified file 'meliae/tests/test__loader.py'
--- a/meliae/tests/test__loader.py 2009-12-28 04:25:27 +0000
+++ b/meliae/tests/test__loader.py 2009-12-28 05:06:17 +0000
@@ -133,6 +133,18 @@
self.assertEqual(933, moc._test_lookup(933L+1024L))
self.assertEqual(933, moc._test_lookup(933L+2**32-1))
+ def test__len__(self):
+ moc = _loader.MemObjectCollection()
+ self.assertEqual(0, len(moc))
+ moc.add(0, 'foo', 100)
+ self.assertEqual(1, len(moc))
+ moc.add(1024, 'foo', 100)
+ self.assertEqual(2, len(moc))
+ del moc[0]
+ self.assertEqual(1, len(moc))
+ del moc[1024]
+ self.assertEqual(0, len(moc))
+
def test__lookup_collide(self):
moc = _loader.MemObjectCollection()
self.assertEqual(1023, moc._table_mask)
@@ -172,6 +184,12 @@
self.assertRaises(KeyError, get, 1024)
self.assertTrue(mop is moc[mop])
+ def test_get(self):
+ moc = _loader.MemObjectCollection()
+ self.assertEqual(None, moc.get(0, None))
+ moc.add(0, 'foo', 100)
+ self.assertEqual(100, moc.get(0, None).size)
+
def test__delitem__(self):
moc = _loader.MemObjectCollection()
def get(offset):
@@ -203,6 +221,26 @@
self.assertEqual(1024, mop.address)
self.assertEqual(1124, mop.size)
+ def test_itervalues(self):
+ moc = _loader.MemObjectCollection()
+ moc.add(0, 'bar', 100)
+ moc.add(1024, 'baz', 102)
+ moc.add(512, 'bing', 103)
+ self.assertEqual([0, 1024, 512], [x.address for x in moc.itervalues()])
+ del moc[0]
+ self.assertEqual([1024, 512], [x.address for x in moc.itervalues()])
+
+ def test_iteritems(self):
+ moc = _loader.MemObjectCollection()
+ moc.add(0, 'bar', 100)
+ moc.add(1024, 'baz', 102)
+ moc.add(512, 'bing', 103)
+ self.assertEqual([(0, 0), (1024, 1024), (512, 512)],
+ [(k, v.address) for k,v in moc.iteritems()])
+ del moc[0]
+ self.assertEqual([(1024, 1024), (512, 512)],
+ [(k, v.address) for k,v in moc.iteritems()])
+
class Test_MemObjectProxy(tests.TestCase):
@@ -280,3 +318,12 @@
self.assertEqual('foo', mop0.type_str)
self.assertEqual(255, mop255.address)
self.assertEqual('baz', mop255.type_str)
+
+ def test_total_size(self):
+ mop = self.moc[0]
+ self.assertEqual(0, mop.total_size)
+ mop.total_size = 10245678
+ self.assertEqual(10245678, mop.total_size)
+ mop.total_size = (2**31+1)
+ self.assertEqual(2**31+1, mop.total_size)
+
More information about the bazaar-commits
mailing list