Rev 2108: Use deques, share more code between full repository fetching and svn-import. in http://people.samba.org/bzr/jelmer/bzr-svn/0.5
Jelmer Vernooij
jelmer at samba.org
Sun Nov 30 19:56:52 GMT 2008
At http://people.samba.org/bzr/jelmer/bzr-svn/0.5
------------------------------------------------------------
revno: 2108
revision-id: jelmer at samba.org-20081130195648-i6l2z6pp3jnyhl5e
parent: jelmer at samba.org-20081130172102-ua0z8qa72b4xcvys
committer: Jelmer Vernooij <jelmer at samba.org>
branch nick: 0.5
timestamp: Sun 2008-11-30 20:56:48 +0100
message:
Use deques, share more code between full repository fetching and svn-import.
modified:
convert.py svn2bzr.py-20051018015439-cb4563bff29e632d
fetch.py fetch.py-20060625004942-x2lfaib8ra707a8p-1
revmeta.py revmeta.py-20080901215045-n8a6arqybs9ez5hl-1
=== modified file 'convert.py'
--- a/convert.py 2008-11-29 16:26:45 +0000
+++ b/convert.py 2008-11-30 19:56:48 +0000
@@ -15,6 +15,7 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""Conversion of full repositories."""
+from itertools import tee
import os
from bzrlib import (
@@ -36,7 +37,9 @@
from subvertpy import SubversionException, repos, ERR_STREAM_MALFORMED_DATA
from bzrlib.plugins.svn.branch import SvnBranch
+from bzrlib.plugins.svn.fetch import FetchRevisionFinder
from bzrlib.plugins.svn.format import get_rich_root_format
+from bzrlib.plugins.svn.revmeta import filter_revisions
LATEST_IMPORT_REVISION_FILENAME = "svn-import-revision"
@@ -207,29 +210,20 @@
if to_revnum is None:
to_revnum = source_repos.get_latest_revnum()
mapping = source_repos.get_mapping()
- revmetas = []
existing_branches = {}
deleted = set()
+ it = source_repos._revmeta_provider.iter_all_changes(layout, mapping.is_branch_or_tag, to_revnum, from_revnum, project=project)
+ if create_shared_repo:
+ revfinder = FetchRevisionFinder(source_repos, target_repos, target_repos_is_empty)
+ (it, it_rev) = tee(it)
pb = ui.ui_factory.nested_progress_bar()
try:
- for kind, item in source_repos._revmeta_provider.iter_all_changes(layout, mapping.is_branch_or_tag,
- to_revnum, from_revnum,
- project=project):
+ for kind, item in it:
if kind == "revision":
- revmeta = item
- pb.update("determining revisions to fetch", to_revnum-revmeta.revnum, to_revnum)
- try:
- if revmeta.is_hidden(mapping):
- continue
- # FIXME: mapping = revmeta.get_appropriate_mapping(mapping)
- if target_repos is not None and (target_repos_is_empty or not target_repos.has_revision(revmeta.get_revision_id(mapping))):
- revmetas.append((revmeta, mapping))
- if (not revmeta.branch_path in existing_branches and
- layout.is_branch(revmeta.branch_path, project=project) and
- not contains_parent_path(deleted, revmeta.branch_path)):
- existing_branches[revmeta.branch_path] = SvnBranch(source_repos, revmeta.branch_path, revnum=revmeta.revnum, _skip_check=True)
- except SubversionException, (_, ERR_FS_NOT_DIRECTORY):
- continue
+ if (not item.branch_path in existing_branches and
+ layout.is_branch(item.branch_path, project=project) and
+ not contains_parent_path(deleted, item.branch_path)):
+ existing_branches[item.branch_path] = SvnBranch(source_repos, item.branch_path, revnum=item.revnum, _skip_check=True)
elif kind == "delete":
deleted.add(item)
finally:
@@ -241,7 +235,8 @@
if (target_repos.is_shared() and
getattr(inter, '_supports_revmetas', None) and
inter._supports_revmetas):
- revmetas.reverse()
+ revmetas = revfinder.find_iter(filter_revisions(it_rev),
+ mapping)
inter.fetch(needed=revmetas)
elif all:
inter.fetch()
=== modified file 'fetch.py'
--- a/fetch.py 2008-11-28 17:23:10 +0000
+++ b/fetch.py 2008-11-30 19:56:48 +0000
@@ -23,9 +23,10 @@
from bzrlib.repository import InterRepository
from bzrlib.trace import mutter
+from collections import deque
from cStringIO import StringIO
-from subvertpy import properties
+from subvertpy import properties, SubversionException
from subvertpy.delta import apply_txdelta_handler
from bzrlib.plugins.svn.errors import InvalidFileName
@@ -667,9 +668,22 @@
def needs_fetching(self, revmeta, mapping):
if self.target_is_empty:
return True
- if revmeta.is_hidden(mapping):
- return false
- return not self.target.has_revision(revmeta.get_revision_id(mapping))
+ try:
+ if revmeta.is_hidden(mapping):
+ return False
+ return not self.target.has_revision(revmeta.get_revision_id(mapping))
+ except SubversionException, (_, ERR_FS_NOT_DIRECTORY):
+ return False
+
+ def find_iter(self, iter, mapping):
+ needed = deque()
+ for revmeta in iter:
+ #FIXME: mapping = revmeta.get_appropriate_mapping(mapping)
+ if self.needs_fetching(revmeta, mapping):
+ needed.appendleft((revmeta, mapping))
+ self.checked.add((revmeta.get_foreign_revid(), mapping))
+
+ return needed
def find_all(self, mapping, pb=None):
"""Find all revisions from the source repository that are not
@@ -677,19 +691,8 @@
:return: List with revmeta, mapping tuples to fetch
"""
- needed = []
from_revnum = self.source.get_latest_revnum()
- for revmeta in self.source._revmeta_provider.iter_all_revisions(self.source.get_layout(), check_unusual_path=mapping.is_branch_or_tag, from_revnum=from_revnum, pb=pb):
- #FIXME: mapping = revmeta.get_appropriate_mapping(mapping)
- if pb:
- pb.update("determining revisions to fetch",
- from_revnum-revmeta.revnum, from_revnum)
- if self.needs_fetching(revmeta, mapping):
- needed.append((revmeta, mapping))
- self.checked.add((revmeta.get_foreign_revid(), mapping))
-
- needed.reverse()
- return needed
+ return self.find_iter(self.source._revmeta_provider.iter_all_revisions(self.source.get_layout(), check_unusual_path=mapping.is_branch_or_tag, from_revnum=from_revnum, pb=pb), mapping)
def find_until(self, foreign_revid, mapping, find_ghosts=False, pb=None,
project=None):
@@ -703,7 +706,7 @@
def check_revid(foreign_revid, mapping, project=None):
if (foreign_revid, mapping) in self.checked:
return []
- revmetas = []
+ revmetas = deque()
(uuid, branch_path, revnum) = foreign_revid
# TODO: Do binary search to find first revision to fetch if
# fetch_ghosts=False ?
@@ -717,7 +720,7 @@
# This revision (and its ancestry) has already been checked
break
if self.needs_fetching(revmeta, mapping):
- revmetas.append((revmeta, mapping))
+ revmetas.appendleft((revmeta, mapping))
for p in revmeta.get_rhs_parents(mapping):
try:
foreign_revid, mapping = self.source.lookup_revision_id(p, project=project)
@@ -728,7 +731,6 @@
elif not find_ghosts:
break
self.checked.add((revmeta.get_foreign_revid(), mapping))
- revmetas.reverse()
return revmetas
needed = check_revid(foreign_revid, mapping, project)
=== modified file 'revmeta.py'
--- a/revmeta.py 2008-11-30 17:02:03 +0000
+++ b/revmeta.py 2008-11-30 19:56:48 +0000
@@ -580,8 +580,7 @@
class RevisionMetadataBranch(object):
"""Describes a Bazaar-like branch in a Subversion repository."""
- def __init__(self, revmeta_provider=None,
- history_limit=None):
+ def __init__(self, revmeta_provider=None, history_limit=None):
self._revs = []
self._revnums = []
self._history_limit = history_limit
@@ -715,7 +714,8 @@
unusual_history = {}
metabranches_history = {}
unusual = set()
- for (paths, revnum, revprops) in self._provider._log.iter_changes(self.prefixes, self.from_revnum, self.to_revnum, pb=pb):
+ for (paths, revnum, revprops) in self._provider._log.iter_changes(
+ self.prefixes, self.from_revnum, self.to_revnum, pb=pb):
bps = {}
deletes = []
if pb:
@@ -778,6 +778,12 @@
yield "revision", revmeta
+def filter_revisions(it):
+ for kind, rev in it:
+ if kind == "revision":
+ yield rev
+
+
class RevisionMetadataProvider(object):
"""A RevisionMetadata provider."""
@@ -912,10 +918,7 @@
Layout decides which ones to pick up.
"""
- for kind, rev in self.iter_all_changes(layout, check_unusual_path,
- from_revnum, to_revnum, project, pb):
- if kind == "revision":
- yield rev
+ return filter_revisions(self.iter_all_changes(layout, check_unusual_path, from_revnum, to_revnum, project, pb))
def iter_all_changes(self, layout, check_unusual_path, from_revnum,
to_revnum=0, project=None, pb=None):
More information about the bazaar-commits
mailing list