Rev 4712: (jam) Address bug #343218 by batching file texts request during in file:///home/pqm/archives/thelove/bzr/2.0/

[Canonical.com Patch Queue Manager]

At file:///home/pqm/archives/thelove/bzr/2.0/

------------------------------------------------------------
revno: 4712 [merge]
revision-id: pqm at pqm.ubuntu.com-20091218170035-kas8ryyoom4x2uti
parent: pqm at pqm.ubuntu.com-20091217060447-srg2pdrib5iij6uf
parent: john at arbash-meinel.com-20091218053840-d1636x6jf5i3peym
committer: Canonical.com Patch Queue Manager <pqm at pqm.ubuntu.com>
branch nick: 2.0
timestamp: Fri 2009-12-18 17:00:35 +0000
message:
  (jam) Address bug #343218 by batching file texts request during
  	export.
modified:
  NEWS                           NEWS-20050323055033-4e00b5db738777ff
  bzrlib/export/dir_exporter.py  dir_exporter.py-20051114235828-b51397f56bc7b117
=== modified file 'NEWS'
--- a/NEWS	2009-12-15 16:56:18 +0000
+++ b/NEWS	2009-12-15 22:04:24 +0000
@@ -20,6 +20,11 @@
 Bug Fixes
 *********
 
+* ``bzr export dir`` now requests all file content as a record stream,
+  rather than requsting the file content one file-at-a-time. This can make
+  exporting over the network significantly faster (54min => 9min in one
+  case). (John Arbash Meinel, #343218)
+
 Improvements
 ************
 

=== modified file 'bzrlib/export/dir_exporter.py'
--- a/bzrlib/export/dir_exporter.py	2009-07-29 13:46:55 +0000
+++ b/bzrlib/export/dir_exporter.py	2009-12-18 05:38:40 +0000
@@ -1,4 +1,4 @@
-# Copyright (C) 2005 Canonical Ltd
+# Copyright (C) 2005, 2009 Canonical Ltd
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -52,21 +52,17 @@
                 raise errors.BzrError("Can't export tree to non-empty directory.")
         else:
             raise
+    # Iterate everything, building up the files we will want to export, and
+    # creating the directories and symlinks that we need.
+    # This tracks (file_id, (destination_path, executable))
+    # This matches the api that tree.iter_files_bytes() wants
+    # Note in the case of revision trees, this does trigger a double inventory
+    # lookup, hopefully it isn't too expensive.
+    to_fetch = []
     for dp, ie in _export_iter_entries(tree, subdir):
         fullpath = osutils.pathjoin(dest, dp)
         if ie.kind == "file":
-            if filtered:
-                chunks = tree.get_file_lines(ie.file_id)
-                filters = tree._content_filter_stack(dp)
-                context = ContentFilterContext(dp, tree, ie)
-                contents = filtered_output_bytes(chunks, filters, context)
-                content = ''.join(contents)
-                fileobj = StringIO.StringIO(content)
-            else:
-                fileobj = tree.get_file(ie.file_id)
-            osutils.pumpfile(fileobj, file(fullpath, 'wb'))
-            if tree.is_executable(ie.file_id):
-                os.chmod(fullpath, 0755)
+            to_fetch.append((ie.file_id, (dp, tree.is_executable(ie.file_id))))
         elif ie.kind == "directory":
             os.mkdir(fullpath)
         elif ie.kind == "symlink":
@@ -80,3 +76,21 @@
         else:
             raise errors.BzrError("don't know how to export {%s} of kind %r" %
                (ie.file_id, ie.kind))
+    # The data returned here can be in any order, but we've already created all
+    # the directories
+    flags = os.O_CREAT | os.O_TRUNC | os.O_WRONLY | getattr(os, 'O_BINARY', 0)
+    for (relpath, executable), chunks in tree.iter_files_bytes(to_fetch):
+        if filtered:
+            filters = tree._content_filter_stack(relpath)
+            context = ContentFilterContext(relpath, tree, ie)
+            chunks = filtered_output_bytes(chunks, filters, context)
+        fullpath = osutils.pathjoin(dest, relpath)
+        # We set the mode and let the umask sort out the file info
+        mode = 0666
+        if executable:
+            mode = 0777
+        out = os.fdopen(os.open(fullpath, flags, mode), 'wb')
+        try:
+            out.writelines(chunks)
+        finally:
+            out.close()