Rev 6314: (jelmer) Add a ``Repository.iter_revisions`` HPSS call. (Jelmer Vernooij) in file:///srv/pqm.bazaar-vcs.org/archives/thelove/bzr/%2Btrunk/
Patch Queue Manager
pqm at pqm.ubuntu.com
Mon Nov 28 13:54:34 UTC 2011
At file:///srv/pqm.bazaar-vcs.org/archives/thelove/bzr/%2Btrunk/
------------------------------------------------------------
revno: 6314 [merge]
revision-id: pqm at pqm.ubuntu.com-20111128135433-xoddhlxx9qgi4u5k
parent: pqm at pqm.ubuntu.com-20111128121442-e21724bjf6i6d5xo
parent: jelmer at samba.org-20111128130515-s29pwv14nl5w4mbl
committer: Patch Queue Manager <pqm at pqm.ubuntu.com>
branch nick: +trunk
timestamp: Mon 2011-11-28 13:54:33 +0000
message:
(jelmer) Add a ``Repository.iter_revisions`` HPSS call. (Jelmer Vernooij)
modified:
bzrlib/remote.py remote.py-20060720103555-yeeg2x51vn0rbtdp-1
bzrlib/smart/repository.py repository.py-20061128022038-vr5wy5bubyb8xttk-1
bzrlib/smart/request.py request.py-20061108095550-gunadhxmzkdjfeek-1
bzrlib/tests/blackbox/test_log.py test_log.py-20060112090212-78f6ea560c868e24
bzrlib/tests/blackbox/test_sign_my_commits.py test_sign_my_commits.py-20060215152957-270238a1ffacc841
bzrlib/tests/test_remote.py test_remote.py-20060720103555-yeeg2x51vn0rbtdp-2
bzrlib/tests/test_smart.py test_smart.py-20061122024551-ol0l0o0oofsu9b3t-2
doc/en/release-notes/bzr-2.5.txt bzr2.5.txt-20110708125756-587p0hpw7oke4h05-1
=== modified file 'bzrlib/remote.py'
--- a/bzrlib/remote.py 2011-11-28 11:49:43 +0000
+++ b/bzrlib/remote.py 2011-11-28 13:05:15 +0000
@@ -15,6 +15,7 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
import bz2
+import zlib
from bzrlib import (
bencode,
@@ -51,6 +52,7 @@
from bzrlib.revision import NULL_REVISION
from bzrlib.revisiontree import InventoryRevisionTree
from bzrlib.repository import RepositoryWriteLockResult, _LazyListJoin
+from bzrlib.serializer import format_registry as serializer_format_registry
from bzrlib.trace import mutter, note, warning, log_exception_quietly
@@ -2229,10 +2231,69 @@
self._ensure_real()
return self._real_repository.texts
+ def _iter_revisions_rpc(self, revision_ids):
+ body = "\n".join(revision_ids)
+ path = self.bzrdir._path_for_remote_call(self._client)
+ response_tuple, response_handler = (
+ self._call_with_body_bytes_expecting_body(
+ "Repository.iter_revisions", (path, ), body))
+ if response_tuple[0] != "ok":
+ raise errors.UnexpectedSmartServerResponse(response_tuple)
+ serializer_format = response_tuple[1]
+ serializer = serializer_format_registry.get(serializer_format)
+ byte_stream = response_handler.read_streamed_body()
+ decompressor = zlib.decompressobj()
+ chunks = []
+ for bytes in byte_stream:
+ chunks.append(decompressor.decompress(bytes))
+ if decompressor.unused_data != "":
+ chunks.append(decompressor.flush())
+ yield serializer.read_revision_from_string("".join(chunks))
+ unused = decompressor.unused_data
+ decompressor = zlib.decompressobj()
+ chunks = [decompressor.decompress(unused)]
+ chunks.append(decompressor.flush())
+ text = "".join(chunks)
+ if text != "":
+ yield serializer.read_revision_from_string("".join(chunks))
+
@needs_read_lock
def get_revisions(self, revision_ids):
- self._ensure_real()
- return self._real_repository.get_revisions(revision_ids)
+ if revision_ids is None:
+ revision_ids = self.all_revision_ids()
+ else:
+ for rev_id in revision_ids:
+ if not rev_id or not isinstance(rev_id, basestring):
+ raise errors.InvalidRevisionId(
+ revision_id=rev_id, branch=self)
+ try:
+ missing = set(revision_ids)
+ revs = {}
+ for rev in self._iter_revisions_rpc(revision_ids):
+ missing.remove(rev.revision_id)
+ revs[rev.revision_id] = rev
+ except errors.UnknownSmartMethod:
+ self._ensure_real()
+ return self._real_repository.get_revisions(revision_ids)
+ for fallback in self._fallback_repositories:
+ if not missing:
+ break
+ for revid in list(missing):
+ # XXX JRV 2011-11-20: It would be nice if there was a
+ # public method on Repository that could be used to query
+ # for revision objects *without* failing completely if one
+ # was missing. There is VersionedFileRepository._iter_revisions,
+ # but unfortunately that's private and not provided by
+ # all repository implementations.
+ try:
+ revs[revid] = fallback.get_revision(revid)
+ except errors.NoSuchRevision:
+ pass
+ else:
+ missing.remove(revid)
+ if missing:
+ raise errors.NoSuchRevision(self, list(missing)[0])
+ return [revs[revid] for revid in revision_ids]
def supports_rich_root(self):
return self._format.rich_root_data
=== modified file 'bzrlib/smart/repository.py'
--- a/bzrlib/smart/repository.py 2011-11-28 03:38:43 +0000
+++ b/bzrlib/smart/repository.py 2011-11-28 11:34:18 +0000
@@ -22,6 +22,7 @@
import sys
import tempfile
import threading
+import zlib
from bzrlib import (
bencode,
@@ -1101,3 +1102,39 @@
finally:
self._repository.unlock()
return SuccessfulSmartServerResponse(("ok", ), )
+
+
+class SmartServerRepositoryIterRevisions(SmartServerRepositoryRequest):
+ """Stream a list of revisions.
+
+ The client sends a list of newline-separated revision ids in the
+ body of the request and the server replies with the serializer format,
+ and a stream of bzip2-compressed revision texts (using the specified
+ serializer format).
+
+ Any revisions the server does not have are omitted from the stream.
+
+ New in 2.5.
+ """
+
+ def do_repository_request(self, repository):
+ self._repository = repository
+ # Signal there is a body
+ return None
+
+ def do_body(self, body_bytes):
+ revision_ids = body_bytes.split("\n")
+ return SuccessfulSmartServerResponse(
+ ('ok', self._repository.get_serializer_format()),
+ body_stream=self.body_stream(self._repository, revision_ids))
+
+ def body_stream(self, repository, revision_ids):
+ self._repository.lock_read()
+ try:
+ for record in repository.revisions.get_record_stream(
+ [(revid,) for revid in revision_ids], 'unordered', True):
+ if record.storage_kind == 'absent':
+ continue
+ yield zlib.compress(record.get_bytes_as('fulltext'))
+ finally:
+ self._repository.unlock()
=== modified file 'bzrlib/smart/request.py'
--- a/bzrlib/smart/request.py 2011-11-28 03:38:43 +0000
+++ b/bzrlib/smart/request.py 2011-11-28 11:34:18 +0000
@@ -676,6 +676,9 @@
'Repository.get_stream_1.19', 'bzrlib.smart.repository',
'SmartServerRepositoryGetStream_1_19')
request_handlers.register_lazy(
+ 'Repository.iter_revisions', 'bzrlib.smart.repository',
+ 'SmartServerRepositoryIterRevisions')
+request_handlers.register_lazy(
'Repository.pack', 'bzrlib.smart.repository',
'SmartServerRepositoryPack')
request_handlers.register_lazy(
=== modified file 'bzrlib/tests/blackbox/test_log.py'
--- a/bzrlib/tests/blackbox/test_log.py 2011-11-23 11:22:56 +0000
+++ b/bzrlib/tests/blackbox/test_log.py 2011-11-23 12:00:17 +0000
@@ -1069,7 +1069,7 @@
# being too low. If rpc_count increases, more network roundtrips have
# become necessary for this use case. Please do not adjust this number
# upwards without agreement from bzr's network support maintainers.
- self.assertLength(15, self.hpss_calls)
+ self.assertLength(10, self.hpss_calls)
def test_verbose_log(self):
self.setup_smart_server_with_call_log()
@@ -1084,4 +1084,4 @@
# being too low. If rpc_count increases, more network roundtrips have
# become necessary for this use case. Please do not adjust this number
# upwards without agreement from bzr's network support maintainers.
- self.assertLength(20, self.hpss_calls)
+ self.assertLength(19, self.hpss_calls)
=== modified file 'bzrlib/tests/blackbox/test_sign_my_commits.py'
--- a/bzrlib/tests/blackbox/test_sign_my_commits.py 2011-11-28 11:17:05 +0000
+++ b/bzrlib/tests/blackbox/test_sign_my_commits.py 2011-11-28 13:05:15 +0000
@@ -164,7 +164,7 @@
# being too low. If rpc_count increases, more network roundtrips have
# become necessary for this use case. Please do not adjust this number
# upwards without agreement from bzr's network support maintainers.
- self.assertLength(53, self.hpss_calls)
+ self.assertLength(54, self.hpss_calls)
def test_verify_commits(self):
self.setup_smart_server_with_call_log()
@@ -186,6 +186,6 @@
# The number of readv requests seems to vary depending on the generated
# repository and how well it compresses, so allow for a bit of
# variation:
- if len(self.hpss_calls) not in (21, 22):
- self.fail("Incorrect length: wanted 21 or 22, got %d for %r" % (
+ if len(self.hpss_calls) not in (20, 21):
+ self.fail("Incorrect length: wanted 20 or 21, got %d for %r" % (
len(self.hpss_calls), self.hpss_calls))
=== modified file 'bzrlib/tests/test_remote.py'
--- a/bzrlib/tests/test_remote.py 2011-11-28 11:49:43 +0000
+++ b/bzrlib/tests/test_remote.py 2011-11-28 13:05:15 +0000
@@ -25,6 +25,7 @@
import bz2
from cStringIO import StringIO
+import zlib
from bzrlib import (
branch,
@@ -48,6 +49,7 @@
BzrDirFormat,
RemoteBzrProber,
)
+from bzrlib.chk_serializer import chk_bencode_serializer
from bzrlib.remote import (
RemoteBranch,
RemoteBranchFormat,
@@ -57,7 +59,10 @@
RemoteRepositoryFormat,
)
from bzrlib.repofmt import groupcompress_repo, knitpack_repo
-from bzrlib.revision import NULL_REVISION
+from bzrlib.revision import (
+ NULL_REVISION,
+ Revision,
+ )
from bzrlib.smart import medium, request
from bzrlib.smart.client import _SmartClient
from bzrlib.smart.repository import (
@@ -2620,6 +2625,43 @@
self.assertEqual({'rev1': ('null:',)}, graph.get_parent_map(['rev1']))
+class TestRepositoryGetRevisions(TestRemoteRepository):
+
+ def test_hpss_missing_revision(self):
+ transport_path = 'quack'
+ repo, client = self.setup_fake_client_and_repository(transport_path)
+ client.add_success_response_with_body(
+ '', 'ok', '10')
+ self.assertRaises(errors.NoSuchRevision, repo.get_revisions,
+ ['somerev1', 'anotherrev2'])
+ self.assertEqual(
+ [('call_with_body_bytes_expecting_body', 'Repository.iter_revisions',
+ ('quack/', ), "somerev1\nanotherrev2")],
+ client._calls)
+
+ def test_hpss_get_single_revision(self):
+ transport_path = 'quack'
+ repo, client = self.setup_fake_client_and_repository(transport_path)
+ somerev1 = Revision("somerev1")
+ somerev1.committer = "Joe Committer <joe at example.com>"
+ somerev1.timestamp = 1321828927
+ somerev1.timezone = -60
+ somerev1.inventory_sha1 = "691b39be74c67b1212a75fcb19c433aaed903c2b"
+ somerev1.message = "Message"
+ body = zlib.compress(chk_bencode_serializer.write_revision_to_string(
+ somerev1))
+ # Split up body into two bits to make sure the zlib compression object
+ # gets data fed twice.
+ client.add_success_response_with_body(
+ [body[:10], body[10:]], 'ok', '10')
+ revs = repo.get_revisions(['somerev1'])
+ self.assertEquals(revs, [somerev1])
+ self.assertEqual(
+ [('call_with_body_bytes_expecting_body', 'Repository.iter_revisions',
+ ('quack/', ), "somerev1")],
+ client._calls)
+
+
class TestRepositoryGetRevisionGraph(TestRemoteRepository):
def test_null_revision(self):
=== modified file 'bzrlib/tests/test_smart.py'
--- a/bzrlib/tests/test_smart.py 2011-11-28 03:38:43 +0000
+++ b/bzrlib/tests/test_smart.py 2011-11-28 11:34:18 +0000
@@ -25,6 +25,7 @@
"""
import bz2
+import zlib
from bzrlib import (
branch as _mod_branch,
@@ -1724,6 +1725,50 @@
request.execute('stacked', 1, (3, r3)))
+class TestSmartServerRepositoryIterRevisions(
+ tests.TestCaseWithMemoryTransport):
+
+ def test_basic(self):
+ backing = self.get_transport()
+ request = smart_repo.SmartServerRepositoryIterRevisions(backing)
+ tree = self.make_branch_and_memory_tree('.', format='2a')
+ tree.lock_write()
+ tree.add('')
+ tree.commit('1st commit', rev_id="rev1")
+ tree.commit('2nd commit', rev_id="rev2")
+ tree.unlock()
+
+ self.assertIs(None, request.execute(''))
+ response = request.do_body("rev1\nrev2")
+ self.assertTrue(response.is_successful())
+ # Format 2a uses serializer format 10
+ self.assertEquals(response.args, ("ok", "10"))
+
+ self.addCleanup(tree.branch.lock_read().unlock)
+ entries = [zlib.compress(record.get_bytes_as("fulltext")) for record in
+ tree.branch.repository.revisions.get_record_stream(
+ [("rev1", ), ("rev2", )], "unordered", True)]
+
+ contents = "".join(response.body_stream)
+ self.assertTrue(contents in (
+ "".join([entries[0], entries[1]]),
+ "".join([entries[1], entries[0]])))
+
+ def test_missing(self):
+ backing = self.get_transport()
+ request = smart_repo.SmartServerRepositoryIterRevisions(backing)
+ tree = self.make_branch_and_memory_tree('.', format='2a')
+
+ self.assertIs(None, request.execute(''))
+ response = request.do_body("rev1\nrev2")
+ self.assertTrue(response.is_successful())
+ # Format 2a uses serializer format 10
+ self.assertEquals(response.args, ("ok", "10"))
+
+ contents = "".join(response.body_stream)
+ self.assertEquals(contents, "")
+
+
class GetStreamTestBase(tests.TestCaseWithMemoryTransport):
def make_two_commit_repo(self):
@@ -2390,6 +2435,8 @@
smart_repo.SmartServerRepositoryGetStream)
self.assertHandlerEqual('Repository.get_stream_1.19',
smart_repo.SmartServerRepositoryGetStream_1_19)
+ self.assertHandlerEqual('Repository.iter_revisions',
+ smart_repo.SmartServerRepositoryIterRevisions)
self.assertHandlerEqual('Repository.has_revision',
smart_repo.SmartServerRequestHasRevision)
self.assertHandlerEqual('Repository.insert_stream',
=== modified file 'doc/en/release-notes/bzr-2.5.txt'
--- a/doc/en/release-notes/bzr-2.5.txt 2011-11-28 11:49:43 +0000
+++ b/doc/en/release-notes/bzr-2.5.txt 2011-11-28 13:05:15 +0000
@@ -128,7 +128,7 @@
``VersionedFileRepository.get_serializer_format``,
``Repository.all_revision_ids``, ``Repository.start_write_group``,
``Repository.commit_write_group``, ``Repository.abort_write_group``
- ``Repository.check_write_group`` and
+ ``Repository.iter_revisions``, ``Repository.check_write_group`` and
``Repository.add_signature_revision_text``.
(Jelmer Vernooij)
More information about the bazaar-commits
mailing list