Rev 4520: Change Repository._iter_inventory_xmls to avoid buffering *everything*. in http://bazaar.launchpad.net/~jameinel/bzr/1.18-inventory-delta

[John Arbash Meinel]

At http://bazaar.launchpad.net/~jameinel/bzr/1.18-inventory-delta

------------------------------------------------------------
revno: 4520
revision-id: john at arbash-meinel.com-20090728205431-ygav9b3cjp2pbqzp
parent: andrew.bennetts at canonical.com-20090722034058-hr78eofhlcfbnequ
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: 1.18-inventory-delta
timestamp: Tue 2009-07-28 15:54:31 -0500
message:
  Change Repository._iter_inventory_xmls to avoid buffering *everything*.
  
  This doesn't make any guarantees about what will get buffered, but at least it
  now has the option to buffer less than all inventories.
-------------- next part --------------
=== modified file 'bzrlib/repository.py'
--- a/bzrlib/repository.py	2009-07-22 03:40:58 +0000
+++ b/bzrlib/repository.py	2009-07-28 20:54:31 +0000
@@ -2208,6 +2208,10 @@
 
     def _iter_inventory_xmls(self, revision_ids, ordering='unordered'):
         keys = [(revision_id,) for revision_id in revision_ids]
+        if not keys:
+            return
+        key_iter = iter(keys)
+        next_key = key_iter.next()
         stream = self.inventories.get_record_stream(keys, ordering, True)
         text_chunks = {}
         for record in stream:
@@ -2215,9 +2219,16 @@
                 text_chunks[record.key] = record.get_bytes_as('chunked')
             else:
                 raise errors.NoSuchRevision(self, record.key)
-        for key in keys:
-            chunks = text_chunks.pop(key)
-            yield ''.join(chunks), key[-1]
+            while next_key in text_chunks:
+                chunks = text_chunks.pop(next_key)
+                yield ''.join(chunks), next_key[-1]
+                try:
+                    next_key = key_iter.next()
+                except StopIteration:
+                    # We still want to fully consume the get_record_stream,
+                    # just in case it is not actually finished at this point
+                    next_key = None
+                    break
 
     def deserialise_inventory(self, revision_id, xml):
         """Transform the xml into an inventory object.
@@ -4219,18 +4230,9 @@
             from_format.network_name() == self.to_format.network_name()):
             raise AssertionError(
                 "this case should be handled by GroupCHKStreamSource")
-        elif (not from_format.supports_chks):
-            # Source repository doesn't support chks. So we can transmit the
-            # inventories 'as-is' and either they are just accepted on the
-            # target, or the Sink will properly convert it.
-            # (XXX: this assumes that all non-chk formats are understood as-is
-            # by any Sink, but that presumably isn't true for foreign repo
-            # formats added by bzr-svn etc?)
-            return self._get_simple_inventory_stream(revision_ids,
-                    missing=missing)
         else:
-            # Make chk->non-chk (and chk with different serializers) fetch:
-            # copy the inventories as (format-neutral) inventory deltas.
+            # Any time we switch serializations, we want to use an
+            # inventory-delta based approach.
             return self._get_convertable_inventory_stream(revision_ids,
                     fulltexts=missing)