Rev 1: Write a Local Transport implementation that can use mmap to handle readv() requests. in http://bzr.arbash-meinel.com/plugins/mmap_readv

[John Arbash Meinel]

At http://bzr.arbash-meinel.com/plugins/mmap_readv

------------------------------------------------------------
revno: 1
revision-id: john at arbash-meinel.com-20090206152240-wz21d1l122su5dj9
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: mmap_readv
timestamp: Fri 2009-02-06 09:22:40 -0600
message:
  Write a Local Transport implementation that can use mmap to handle readv() requests.
-------------- next part --------------
=== added file '__init__.py'
--- a/__init__.py	1970-01-01 00:00:00 +0000
+++ b/__init__.py	2009-02-06 15:22:40 +0000
@@ -0,0 +1,107 @@
+# Copyright (C) 2009 Canonical Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+"""Use mmap to respond to readv requests over local connections instead of
+open+seek+read.
+"""
+
+import os
+
+from bzrlib import errors, osutils, trace, transport
+from bzrlib.transport import local
+from bzrlib.lazy_import import lazy_import
+lazy_import(globals(), """
+import mmap
+""")
+
+
+_mmap_cache = {}
+_first = True
+
+
+class MMapLocalTransport(local.LocalTransport):
+    """An implementation for local access which uses mmap for readv() requests.
+
+    This is otherwise identical to a LocalTransport.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(MMapLocalTransport, self).__init__(*args, **kwargs)
+        global _first
+        if _first:
+            _first = False
+            trace.note('Using MMapLocalTransport')
+
+    # XXX: LocalTransport.clone() doesn't use self.__class__, so do it here
+    def clone(self, offset=None):
+        """Return a new LocalTransport with root at self.base + offset
+        Because the local filesystem does not require a connection,
+        we can just return a new object.
+        """
+        if offset is None:
+            return self.__class__(self.base)
+        else:
+            abspath = self.abspath(offset)
+            if abspath == 'file://':
+                # fix upwalk for UNC path
+                # when clone from //HOST/path updir recursively
+                # we should stop at least at //HOST part
+                abspath = self.base
+            return self.__class__(abspath)
+
+    def _get_mmap(self, relpath):
+        canonical_url = self.abspath(relpath)
+        if canonical_url in transport._file_streams:
+            transport._file_streams[canonical_url].flush()
+        if canonical_url in _mmap_cache:
+            return _mmap_cache[canonical_url]
+        path = self._abspath(relpath)
+        try:
+            fileno = os.open(path, os.O_RDONLY | osutils.O_BINARY)
+        except (IOError, OSError),e:
+            self._translate_error(e, path)
+        mapped = mmap.mmap(fileno, 0, access=mmap.ACCESS_READ)
+        os.close(fileno)
+        _mmap_cache[canonical_url] = mapped
+        return mapped
+
+    def _readv(self, relpath, offsets):
+        """See Transport._readv()"""
+        mapped = self._get_mmap(relpath)
+        # For this first implementation, we don't cache the mmaps, we just
+        # don't require the full open() for the files.
+        for start, length in offsets:
+            bytes = mapped[start:start+length]
+            if len(bytes) != length:
+                raise errors.ShortReadvError(relpath, start, length,
+                                             actual=len(bytes))
+            yield start, bytes
+
+
+# transport.register_transport('file://', MMapLocalTransport)
+
+class MMapLocalURLServer(local.LocalURLServer):
+
+    def tearDown(self):
+        _mmap_cache.clear()
+        super(MMapLocalURLServer, self).tearDown()
+
+
+def get_test_permutations():
+    """Return the permutations to be used in testing."""
+    return [
+            (MMapLocalTransport, MMapLocalURLServer),
+            ]