Rev 107: pre-allocate the big dicts, if we are going to run out of memory, do it early. in http://bazaar.launchpad.net/~meliae-dev/meliae/trunk

[John Arbash Meinel]

At http://bazaar.launchpad.net/~meliae-dev/meliae/trunk

------------------------------------------------------------
revno: 107
revision-id: john at arbash-meinel.com-20091023034828-gsk1wi0b4uqhmctw
parent: john at arbash-meinel.com-20091023034724-fs6w0z6r2g2xk19x
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: trunk
timestamp: Thu 2009-10-22 22:48:28 -0500
message:
  pre-allocate the big dicts, if we are going to run out of memory, do it early.
-------------- next part --------------
=== modified file 'meliae/loader.py'
--- a/meliae/loader.py	2009-10-22 22:05:00 +0000
+++ b/meliae/loader.py	2009-10-23 03:48:28 +0000
@@ -200,19 +200,23 @@
 
     def compute_referrers(self):
         """For each object, figure out who is referencing it."""
-        referrers = {} # From address => [referred from]
-        id_cache = {}
-        unique_address = id_cache.setdefault 
+        referrers = dict.fromkeys(self.objs, None)
+        id_cache = dict((obj.address, obj.address) for obj in
+                        self.objs.itervalues())
         total = len(self.objs)
         for idx, obj in enumerate(self.objs.itervalues()):
             if self.show_progress and idx & 0x1ff == 0:
                 sys.stderr.write('compute referrers %8d / %8d        \r'
                                  % (idx, total))
             address = obj.address
-            address = unique_address(address, address)
             for ref in obj.ref_list:
-                ref = unique_address(ref, ref)
-                refs = referrers.get(ref, None)
+                try:
+                    ref = id_cache[ref]
+                except KeyError:
+                    # Reference to something outside this set of objects.
+                    # Doesn't matter what it is, we won't be updating it.
+                    continue
+                refs = referrers[ref]
                 # This is ugly, so it should be explained.
                 # To save memory pressure, referrers will point to one of 3
                 # types.
@@ -247,7 +251,9 @@
             except KeyError:
                 obj.referrers = ()
             else:
-                if type(refs) is int:
+                if refs is None:
+                    obj.referrers = ()
+                elif type(refs) is int:
                     obj.referrers = (refs,)
                 else:
                     obj.referrers = refs