Rev 4903: (jam) Change the protocol3 buffering code to use a size buffer rather in file:///home/pqm/archives/thelove/bzr/%2Btrunk/

[Canonical.com Patch Queue Manager]

At file:///home/pqm/archives/thelove/bzr/%2Btrunk/

------------------------------------------------------------
revno: 4903 [merge]
revision-id: pqm at pqm.ubuntu.com-20091216181126-rzour6b702sb9sev
parent: pqm at pqm.ubuntu.com-20091216113900-alk0tw723aqrbn6x
parent: john at arbash-meinel.com-20091216172749-sgr2ehcbi9efmauv
committer: Canonical.com Patch Queue Manager <pqm at pqm.ubuntu.com>
branch nick: +trunk
timestamp: Wed 2009-12-16 18:11:26 +0000
message:
  (jam) Change the protocol3 buffering code to use a size buffer rather
  	than a call count buffer
modified:
  bzrlib/smart/protocol.py       protocol.py-20061108035435-ot0lstk2590yqhzr-1
  bzrlib/tests/test_smart_transport.py test_ssh_transport.py-20060608202016-c25gvf1ob7ypbus6-2
=== modified file 'bzrlib/smart/protocol.py'
--- a/bzrlib/smart/protocol.py	2009-12-14 16:16:05 +0000
+++ b/bzrlib/smart/protocol.py	2009-12-16 18:11:26 +0000
@@ -1,4 +1,4 @@
-# Copyright (C) 2006, 2007 Canonical Ltd
+# Copyright (C) 2006, 2007, 2008, 2009 Canonical Ltd
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -1066,9 +1066,11 @@
 class _ProtocolThreeEncoder(object):
 
     response_marker = request_marker = MESSAGE_VERSION_THREE
+    BUFFER_SIZE = 1024*1024 # 1 MiB buffer before flushing
 
     def __init__(self, write_func):
         self._buf = []
+        self._buf_len = 0
         self._real_write_func = write_func
 
     def _write_func(self, bytes):
@@ -1081,13 +1083,15 @@
         #       Note that osutils.send_all always sends 64kB chunks anyway, so
         #       we might just push out smaller bits at a time?
         self._buf.append(bytes)
-        if len(self._buf) > 100:
+        self._buf_len += len(bytes)
+        if self._buf_len > self.BUFFER_SIZE:
             self.flush()
 
     def flush(self):
         if self._buf:
             self._real_write_func(''.join(self._buf))
             del self._buf[:]
+            self._buf_len = 0
 
     def _serialise_offsets(self, offsets):
         """Serialise a readv offset list."""

=== modified file 'bzrlib/tests/test_smart_transport.py'
--- a/bzrlib/tests/test_smart_transport.py	2009-10-27 06:30:37 +0000
+++ b/bzrlib/tests/test_smart_transport.py	2009-12-16 17:27:49 +0000
@@ -2892,13 +2892,13 @@
         self.assertWriteCount(1)
 
     def test_send_response_with_body_stream_flushes_buffers_sometimes(self):
-        """When there are many chunks (>100), multiple writes will occur rather
+        """When there are many bytes (>1MB), multiple writes will occur rather
         than buffering indefinitely.
         """
-        # Construct a response with stream with 40 chunks in it.  Every chunk
-        # triggers 3 buffered writes, so we expect > 100 buffered writes, but <
-        # 200.
-        body_stream = ['chunk %d' % count for count in range(40)]
+        # Construct a response with stream with ~1.5MB in it. This should
+        # trigger 2 writes, but not 3
+        onekib = '12345678' * 128
+        body_stream = [onekib] * (1024 + 512)
         response = _mod_request.SuccessfulSmartServerResponse(
             ('arg', 'arg'), body_stream=body_stream)
         self.responder.send_response(response)