Rev 162: Get some more sizes correct. in http://bazaar.launchpad.net/~meliae-dev/meliae/trunk
John Arbash Meinel
john at arbash-meinel.com
Wed Jul 28 22:13:22 BST 2010
At http://bazaar.launchpad.net/~meliae-dev/meliae/trunk
------------------------------------------------------------
revno: 162
revision-id: john at arbash-meinel.com-20100728211306-czkdksnp4am5ljii
parent: john at arbash-meinel.com-20100728204449-gbwu4mjn72xcgw4d
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: trunk
timestamp: Wed 2010-07-28 16:13:06 -0500
message:
Get some more sizes correct.
Small bugs in earlier calculations, because we weren't counting all
the PyObject bits, etc.
Also, add __sizeof__ for MemObjectCollection. It isn't 100% accurate
because tp_traverse wouldn't be traversing to all the indirectly
referenced objects. (like the integer addresses, and the referenced
items, etc.)
-------------- next part --------------
=== modified file 'meliae/_intset.pyx'
--- a/meliae/_intset.pyx 2010-07-28 20:36:40 +0000
+++ b/meliae/_intset.pyx 2010-07-28 21:13:06 +0000
@@ -66,7 +66,8 @@
return self._count
def __sizeof__(self):
- my_size = (sizeof(Py_ssize_t)*2 + sizeof(int) + sizeof(int_type*))
+ # PyType *, RefCount, + known attributes
+ my_size = sizeof(IntSet)
if self._array != NULL:
my_size += (sizeof(int_type) * (self._mask + 1))
return my_size
=== modified file 'meliae/_loader.pyx'
--- a/meliae/_loader.pyx 2010-07-16 13:46:42 +0000
+++ b/meliae/_loader.pyx 2010-07-28 21:13:06 +0000
@@ -52,6 +52,13 @@
PyObject *refs[0]
+cdef Py_ssize_t sizeof_RefList(RefList *val):
+ """Determine how many bytes for this ref list. val() can be NULL"""
+ if val == NULL:
+ return 0
+ return sizeof(long) + (sizeof(PyObject *) * val.size)
+
+
cdef object _set_default(object d, object val):
"""Either return the value in the dict, or return 'val'.
@@ -318,6 +325,16 @@
_free_mem_object(self._managed_obj)
self._managed_obj = NULL
+ def __sizeof__(self):
+ my_size = sizeof(_MemObjectProxy)
+ if self._managed_obj != NULL:
+ my_size += sizeof(_MemObject)
+ # XXX: Note that to get the memory dump correct for all this stuff,
+ # We need to walk all the RefList objects and get their size, and
+ # tp_traverse should be walking to all of those referenced
+ # integers, etc.
+ return my_size
+
property address:
"""The identifier for the tracked object."""
def __get__(self):
@@ -604,6 +621,19 @@
def __len__(self):
return self._active
+ def __sizeof__(self):
+ cdef int i
+ cdef _MemObject *cur
+ cdef long my_size
+ my_size = (sizeof(MemObjectCollection)
+ + (sizeof(_MemObject**) * (self._table_mask + 1)))
+ for i from 0 <= i <= self._table_mask:
+ cur = self._table[i]
+ if cur != NULL:
+ my_size += (sizeof_RefList(cur.child_list)
+ + sizeof_RefList(cur.parent_list))
+ return my_size
+
cdef _MemObject** _lookup(self, address) except NULL:
cdef long the_hash
cdef size_t i, n_lookup
@@ -796,7 +826,6 @@
PyMem_Free(old_table)
return new_size
-
def add(self, address, type_str, size, children=(), length=0,
value=None, name=None, parent_list=(), total_size=0):
"""Add a new MemObject to this collection."""
=== modified file 'meliae/tests/test__intset.py'
--- a/meliae/tests/test__intset.py 2010-07-28 20:36:40 +0000
+++ b/meliae/tests/test__intset.py 2010-07-28 21:13:06 +0000
@@ -122,10 +122,18 @@
# will put the object into GC even though it doesn't have any 'object'
# references...
# We could do something with a double-entry check
- self.assertSizeOf(3, iset, extra_size=4, has_gc=False)
+ # Size is:
+ # 1: PyType*
+ # 2: refcnt
+ # 3: vtable*
+ # 4: _count
+ # 5: _mask
+ # 6: _array
+ # 4-byte int _has_singleton
+ self.assertSizeOf(6, iset, extra_size=4, has_gc=False)
iset.add(12345)
# Min allocation is 256 entries
- self.assertSizeOf(3+256, iset, extra_size=4, has_gc=False)
+ self.assertSizeOf(6+256, iset, extra_size=4, has_gc=False)
class TestIDSet(TestIntSet):
=== modified file 'meliae/tests/test__loader.py'
--- a/meliae/tests/test__loader.py 2010-07-16 13:46:42 +0000
+++ b/meliae/tests/test__loader.py 2010-07-28 21:13:06 +0000
@@ -18,6 +18,7 @@
from meliae import (
_loader,
+ _scanner,
warn,
tests,
)
@@ -192,6 +193,29 @@
self.assertEqual([1024, 512], list(moc))
self.assertEqual([1024, 512], list(moc.iterkeys()))
+ def assertSizeOf(self, num_words, obj, extra_size=0, has_gc=True):
+ expected_size = extra_size + num_words * _scanner._word_size
+ if has_gc:
+ expected_size += _scanner._gc_head_size
+ self.assertEqual(expected_size, _scanner.size_of(obj))
+
+ def test__sizeof__base(self):
+ moc = _loader.MemObjectCollection()
+ # Empty table size is 1024 pointers
+ # 1: PyType*
+ # 2: refcnt
+ # 3: vtable*
+ # 4: _table*
+ # 3 4-byte int attributes
+ self.assertSizeOf(4+1024, moc, extra_size=3*4, has_gc=False)
+
+ def test__sizeof__with_reflists(self):
+ moc = _loader.MemObjectCollection()
+ # We should assign the memory for ref-lists to the container. A
+ # ref-list allocates the number of entries + 1
+ moc.add(0, 'foo', 100, children=[1234], parent_list=[3456, 7890])
+ self.assertSizeOf(4+1024+2+3, moc, extra_size=3*4, has_gc=False)
+
class Test_MemObjectProxy(tests.TestCase):
@@ -457,3 +481,34 @@
mop = self.moc.add(3, 'dict', 140, children=[2, 1])
as_dict = mop.refs_as_dict()
self.assertEqual({'a': mop_f}, as_dict)
+
+ def assertSizeOf(self, num_words, obj, extra_size=0, has_gc=True):
+ expected_size = extra_size + num_words * _scanner._word_size
+ if has_gc:
+ expected_size += _scanner._gc_head_size
+ self.assertEqual(expected_size, _scanner.size_of(obj))
+
+ def test__sizeof__(self):
+ mop = self.moc[0]
+ # 1: PyType*
+ # 2: refcnt
+ # 3: collection*
+ # 4: _MemObject*
+ # 5: _managed_obj *
+ # No vtable because we have no cdef functions
+ self.assertSizeOf(5, mop, has_gc=True)
+
+ def test__sizeof__managed(self):
+ mop = self.moc[0]
+ del self.moc[0]
+ # If the underlying object has been removed from the collection, then
+ # the memory is now being managed by the mop itself. So it grows by:
+ # 1: PyObject* address
+ # 2: PyObject* type_str
+ # 3: long size
+ # 4: RefList *child_list
+ # 5: PyObject *value
+ # 6: RefList *parent_list
+ # 7: unsigned long total_size
+ # 8: PyObject *proxy
+ self.assertSizeOf(5+8, mop, has_gc=True)
More information about the bazaar-commits
mailing list