Rev 4809: Stop requiring testtools for sftp use. in file:///home/vila/src/bzr/bugs/516183-test-server/
Vincent Ladeuil
v.ladeuil+lp at free.fr
Wed Feb 3 14:08:56 GMT 2010
At file:///home/vila/src/bzr/bugs/516183-test-server/
------------------------------------------------------------
revno: 4809
revision-id: v.ladeuil+lp at free.fr-20100203140855-yuwq660bb3jayr1d
parent: v.ladeuil+lp at free.fr-20100203140252-gx0y05pyn21gg3mg
committer: Vincent Ladeuil <v.ladeuil+lp at free.fr>
branch nick: 516183-test-server
timestamp: Wed 2010-02-03 15:08:55 +0100
message:
Stop requiring testtools for sftp use.
* bzrlib/transport/sftp.py:
Move the test servers code to tests.stub_sftp.
(get_test_permutations): Import the test servers only when the
tests are run.
* bzrlib/tests/test_transport.py:
(TestSSHConnections.test_bzr_connect_to_bzr_ssh): Get the tests
servers from tests.stub_sftp.
* bzrlib/tests/test_sftp_transport.py:
Get the tests servers from tests.stub_sftp.
* bzrlib/tests/stub_sftp.py:
Fix imports, move the test servers code from transport.sftp.
* bzrlib/tests/blackbox/test_selftest.py:
Fix imports.
* bzrlib/builtins.py:
(cmd_selftest.get_transport_type): Fix the test sftp server
import.
-------------- next part --------------
=== modified file 'NEWS'
--- a/NEWS 2010-02-02 08:36:12 +0000
+++ b/NEWS 2010-02-03 14:08:55 +0000
@@ -14,6 +14,9 @@
Bug Fixes
*********
+* Don't require testtools to use sftp.
+ (Vincent Ladeuil, #516183)
+
* Fix "AttributeError in Inter1and2Helper" during fetch.
(Martin Pool, #513432)
=== modified file 'bzrlib/builtins.py'
--- a/bzrlib/builtins.py 2010-01-21 17:54:58 +0000
+++ b/bzrlib/builtins.py 2010-02-03 14:08:55 +0000
@@ -3444,8 +3444,8 @@
def get_transport_type(typestring):
"""Parse and return a transport specifier."""
if typestring == "sftp":
- from bzrlib.transport.sftp import SFTPAbsoluteServer
- return SFTPAbsoluteServer
+ from bzrlib.tests import stub_sftp
+ return stub_sftp.SFTPAbsoluteServer
if typestring == "memory":
from bzrlib.transport.memory import MemoryServer
return MemoryServer
=== modified file 'bzrlib/tests/blackbox/test_selftest.py'
--- a/bzrlib/tests/blackbox/test_selftest.py 2009-12-22 15:37:23 +0000
+++ b/bzrlib/tests/blackbox/test_selftest.py 2010-02-03 14:08:55 +0000
@@ -16,18 +16,14 @@
"""UI tests for the test framework."""
-import bzrlib.transport
from bzrlib import (
benchmarks,
tests,
)
-from bzrlib.errors import ParamikoNotPresent
from bzrlib.tests import (
- features,
- TestCase,
- TestCaseInTempDir,
- TestSkipped,
- )
+ features,
+ stub_sftp,
+ )
class SelfTestPatch:
@@ -51,7 +47,7 @@
tests.selftest = original_selftest
-class TestOptionsWritingToDisk(TestCaseInTempDir, SelfTestPatch):
+class TestOptionsWritingToDisk(tests.TestCaseInTempDir, SelfTestPatch):
def test_benchmark_runs_benchmark_tests(self):
"""selftest --benchmark should change the suite factory."""
@@ -69,7 +65,7 @@
self.assertEqual(0, len(lines))
-class TestOptions(TestCase, SelfTestPatch):
+class TestOptions(tests.TestCase, SelfTestPatch):
def test_load_list(self):
params = self.get_params_passed_to_core('selftest --load-list foo')
@@ -80,7 +76,7 @@
# version.
self.requireFeature(features.paramiko)
params = self.get_params_passed_to_core('selftest --transport=sftp')
- self.assertEqual(bzrlib.transport.sftp.SFTPAbsoluteServer,
+ self.assertEqual(stub_sftp.SFTPAbsoluteServer,
params[1]["transport"])
def test_transport_set_to_memory(self):
=== modified file 'bzrlib/tests/stub_sftp.py'
--- a/bzrlib/tests/stub_sftp.py 2010-02-03 14:02:52 +0000
+++ b/bzrlib/tests/stub_sftp.py 2010-02-03 14:08:55 +0000
@@ -21,13 +21,22 @@
import os
import paramiko
+import select
+import socket
import sys
+import threading
+import time
from bzrlib import (
osutils,
trace,
- )
-
+ urlutils,
+ )
+from bzrlib.transport import (
+ local,
+ Server,
+ ssh,
+ )
class StubServer (paramiko.ServerInterface):
@@ -231,3 +240,323 @@
# removed: chattr, symlink, readlink
# (nothing in bzr's sftp transport uses those)
+
+# ------------- server test implementation --------------
+
+STUB_SERVER_KEY = """
+-----BEGIN RSA PRIVATE KEY-----
+MIICWgIBAAKBgQDTj1bqB4WmayWNPB+8jVSYpZYk80Ujvj680pOTh2bORBjbIAyz
+oWGW+GUjzKxTiiPvVmxFgx5wdsFvF03v34lEVVhMpouqPAYQ15N37K/ir5XY+9m/
+d8ufMCkjeXsQkKqFbAlQcnWMCRnOoPHS3I4vi6hmnDDeeYTSRvfLbW0fhwIBIwKB
+gBIiOqZYaoqbeD9OS9z2K9KR2atlTxGxOJPXiP4ESqP3NVScWNwyZ3NXHpyrJLa0
+EbVtzsQhLn6rF+TzXnOlcipFvjsem3iYzCpuChfGQ6SovTcOjHV9z+hnpXvQ/fon
+soVRZY65wKnF7IAoUwTmJS9opqgrN6kRgCd3DASAMd1bAkEA96SBVWFt/fJBNJ9H
+tYnBKZGw0VeHOYmVYbvMSstssn8un+pQpUm9vlG/bp7Oxd/m+b9KWEh2xPfv6zqU
+avNwHwJBANqzGZa/EpzF4J8pGti7oIAPUIDGMtfIcmqNXVMckrmzQ2vTfqtkEZsA
+4rE1IERRyiJQx6EJsz21wJmGV9WJQ5kCQQDwkS0uXqVdFzgHO6S++tjmjYcxwr3g
+H0CoFYSgbddOT6miqRskOQF3DZVkJT3kyuBgU2zKygz52ukQZMqxCb1fAkASvuTv
+qfpH87Qq5kQhNKdbbwbmd2NxlNabazPijWuphGTdW0VfJdWfklyS2Kr+iqrs/5wV
+HhathJt636Eg7oIjAkA8ht3MQ+XSl9yIJIS8gVpbPxSw5OMfw0PjVE7tBdQruiSc
+nvuQES5C9BMHjF39LZiGH1iLQy7FgdHyoP+eodI7
+-----END RSA PRIVATE KEY-----
+"""
+
+
+class SocketListener(threading.Thread):
+
+ def __init__(self, callback):
+ threading.Thread.__init__(self)
+ self._callback = callback
+ self._socket = socket.socket()
+ self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ self._socket.bind(('localhost', 0))
+ self._socket.listen(1)
+ self.host, self.port = self._socket.getsockname()[:2]
+ self._stop_event = threading.Event()
+
+ def stop(self):
+ # called from outside this thread
+ self._stop_event.set()
+ # use a timeout here, because if the test fails, the server thread may
+ # never notice the stop_event.
+ self.join(5.0)
+ self._socket.close()
+
+ def run(self):
+ while True:
+ readable, writable_unused, exception_unused = \
+ select.select([self._socket], [], [], 0.1)
+ if self._stop_event.isSet():
+ return
+ if len(readable) == 0:
+ continue
+ try:
+ s, addr_unused = self._socket.accept()
+ # because the loopback socket is inline, and transports are
+ # never explicitly closed, best to launch a new thread.
+ threading.Thread(target=self._callback, args=(s,)).start()
+ except socket.error, x:
+ sys.excepthook(*sys.exc_info())
+ warning('Socket error during accept() within unit test server'
+ ' thread: %r' % x)
+ except Exception, x:
+ # probably a failed test; unit test thread will log the
+ # failure/error
+ sys.excepthook(*sys.exc_info())
+ warning('Exception from within unit test server thread: %r' %
+ x)
+
+
+class SocketDelay(object):
+ """A socket decorator to make TCP appear slower.
+
+ This changes recv, send, and sendall to add a fixed latency to each python
+ call if a new roundtrip is detected. That is, when a recv is called and the
+ flag new_roundtrip is set, latency is charged. Every send and send_all
+ sets this flag.
+
+ In addition every send, sendall and recv sleeps a bit per character send to
+ simulate bandwidth.
+
+ Not all methods are implemented, this is deliberate as this class is not a
+ replacement for the builtin sockets layer. fileno is not implemented to
+ prevent the proxy being bypassed.
+ """
+
+ simulated_time = 0
+ _proxied_arguments = dict.fromkeys([
+ "close", "getpeername", "getsockname", "getsockopt", "gettimeout",
+ "setblocking", "setsockopt", "settimeout", "shutdown"])
+
+ def __init__(self, sock, latency, bandwidth=1.0,
+ really_sleep=True):
+ """
+ :param bandwith: simulated bandwith (MegaBit)
+ :param really_sleep: If set to false, the SocketDelay will just
+ increase a counter, instead of calling time.sleep. This is useful for
+ unittesting the SocketDelay.
+ """
+ self.sock = sock
+ self.latency = latency
+ self.really_sleep = really_sleep
+ self.time_per_byte = 1 / (bandwidth / 8.0 * 1024 * 1024)
+ self.new_roundtrip = False
+
+ def sleep(self, s):
+ if self.really_sleep:
+ time.sleep(s)
+ else:
+ SocketDelay.simulated_time += s
+
+ def __getattr__(self, attr):
+ if attr in SocketDelay._proxied_arguments:
+ return getattr(self.sock, attr)
+ raise AttributeError("'SocketDelay' object has no attribute %r" %
+ attr)
+
+ def dup(self):
+ return SocketDelay(self.sock.dup(), self.latency, self.time_per_byte,
+ self._sleep)
+
+ def recv(self, *args):
+ data = self.sock.recv(*args)
+ if data and self.new_roundtrip:
+ self.new_roundtrip = False
+ self.sleep(self.latency)
+ self.sleep(len(data) * self.time_per_byte)
+ return data
+
+ def sendall(self, data, flags=0):
+ if not self.new_roundtrip:
+ self.new_roundtrip = True
+ self.sleep(self.latency)
+ self.sleep(len(data) * self.time_per_byte)
+ return self.sock.sendall(data, flags)
+
+ def send(self, data, flags=0):
+ if not self.new_roundtrip:
+ self.new_roundtrip = True
+ self.sleep(self.latency)
+ bytes_sent = self.sock.send(data, flags)
+ self.sleep(bytes_sent * self.time_per_byte)
+ return bytes_sent
+
+
+class SFTPServer(Server):
+ """Common code for SFTP server facilities."""
+
+ def __init__(self, server_interface=StubServer):
+ self._original_vendor = None
+ self._homedir = None
+ self._server_homedir = None
+ self._listener = None
+ self._root = None
+ self._vendor = ssh.ParamikoVendor()
+ self._server_interface = server_interface
+ # sftp server logs
+ self.logs = []
+ self.add_latency = 0
+
+ def _get_sftp_url(self, path):
+ """Calculate an sftp url to this server for path."""
+ return 'sftp://foo:bar@%s:%d/%s' % (self._listener.host,
+ self._listener.port, path)
+
+ def log(self, message):
+ """StubServer uses this to log when a new server is created."""
+ self.logs.append(message)
+
+ def _run_server_entry(self, sock):
+ """Entry point for all implementations of _run_server.
+
+ If self.add_latency is > 0.000001 then sock is given a latency adding
+ decorator.
+ """
+ if self.add_latency > 0.000001:
+ sock = SocketDelay(sock, self.add_latency)
+ return self._run_server(sock)
+
+ def _run_server(self, s):
+ ssh_server = paramiko.Transport(s)
+ key_file = osutils.pathjoin(self._homedir, 'test_rsa.key')
+ f = open(key_file, 'w')
+ f.write(STUB_SERVER_KEY)
+ f.close()
+ host_key = paramiko.RSAKey.from_private_key_file(key_file)
+ ssh_server.add_server_key(host_key)
+ server = self._server_interface(self)
+ ssh_server.set_subsystem_handler('sftp', paramiko.SFTPServer,
+ StubSFTPServer, root=self._root,
+ home=self._server_homedir)
+ event = threading.Event()
+ ssh_server.start_server(event, server)
+ event.wait(5.0)
+
+ def start_server(self, backing_server=None):
+ # XXX: TODO: make sftpserver back onto backing_server rather than local
+ # disk.
+ if not (backing_server is None or
+ isinstance(backing_server, local.LocalURLServer)):
+ raise AssertionError(
+ "backing_server should not be %r, because this can only serve the "
+ "local current working directory." % (backing_server,))
+ self._original_vendor = ssh._ssh_vendor_manager._cached_ssh_vendor
+ ssh._ssh_vendor_manager._cached_ssh_vendor = self._vendor
+ if sys.platform == 'win32':
+ # Win32 needs to use the UNICODE api
+ self._homedir = getcwd()
+ else:
+ # But Linux SFTP servers should just deal in bytestreams
+ self._homedir = os.getcwd()
+ if self._server_homedir is None:
+ self._server_homedir = self._homedir
+ self._root = '/'
+ if sys.platform == 'win32':
+ self._root = ''
+ self._listener = SocketListener(self._run_server_entry)
+ self._listener.setDaemon(True)
+ self._listener.start()
+
+ def stop_server(self):
+ self._listener.stop()
+ ssh._ssh_vendor_manager._cached_ssh_vendor = self._original_vendor
+
+ def get_bogus_url(self):
+ """See bzrlib.transport.Server.get_bogus_url."""
+ # this is chosen to try to prevent trouble with proxies, wierd dns, etc
+ # we bind a random socket, so that we get a guaranteed unused port
+ # we just never listen on that port
+ s = socket.socket()
+ s.bind(('localhost', 0))
+ return 'sftp://%s:%s/' % s.getsockname()
+
+
+class SFTPFullAbsoluteServer(SFTPServer):
+ """A test server for sftp transports, using absolute urls and ssh."""
+
+ def get_url(self):
+ """See bzrlib.transport.Server.get_url."""
+ homedir = self._homedir
+ if sys.platform != 'win32':
+ # Remove the initial '/' on all platforms but win32
+ homedir = homedir[1:]
+ return self._get_sftp_url(urlutils.escape(homedir))
+
+
+class SFTPServerWithoutSSH(SFTPServer):
+ """An SFTP server that uses a simple TCP socket pair rather than SSH."""
+
+ def __init__(self):
+ super(SFTPServerWithoutSSH, self).__init__()
+ self._vendor = ssh.LoopbackVendor()
+
+ def _run_server(self, sock):
+ # Re-import these as locals, so that they're still accessible during
+ # interpreter shutdown (when all module globals get set to None, leading
+ # to confusing errors like "'NoneType' object has no attribute 'error'".
+ class FakeChannel(object):
+ def get_transport(self):
+ return self
+ def get_log_channel(self):
+ return 'paramiko'
+ def get_name(self):
+ return '1'
+ def get_hexdump(self):
+ return False
+ def close(self):
+ pass
+
+ server = paramiko.SFTPServer(
+ FakeChannel(), 'sftp', StubServer(self), StubSFTPServer,
+ root=self._root, home=self._server_homedir)
+ try:
+ server.start_subsystem(
+ 'sftp', None, ssh.SocketAsChannelAdapter(sock))
+ except socket.error, e:
+ if (len(e.args) > 0) and (e.args[0] == errno.EPIPE):
+ # it's okay for the client to disconnect abruptly
+ # (bug in paramiko 1.6: it should absorb this exception)
+ pass
+ else:
+ raise
+ except Exception, e:
+ # This typically seems to happen during interpreter shutdown, so
+ # most of the useful ways to report this error are won't work.
+ # Writing the exception type, and then the text of the exception,
+ # seems to be the best we can do.
+ import sys
+ sys.stderr.write('\nEXCEPTION %r: ' % (e.__class__,))
+ sys.stderr.write('%s\n\n' % (e,))
+ server.finish_subsystem()
+
+
+class SFTPAbsoluteServer(SFTPServerWithoutSSH):
+ """A test server for sftp transports, using absolute urls."""
+
+ def get_url(self):
+ """See bzrlib.transport.Server.get_url."""
+ homedir = self._homedir
+ if sys.platform != 'win32':
+ # Remove the initial '/' on all platforms but win32
+ homedir = homedir[1:]
+ return self._get_sftp_url(urlutils.escape(homedir))
+
+
+class SFTPHomeDirServer(SFTPServerWithoutSSH):
+ """A test server for sftp transports, using homedir relative urls."""
+
+ def get_url(self):
+ """See bzrlib.transport.Server.get_url."""
+ return self._get_sftp_url("~/")
+
+
+class SFTPSiblingAbsoluteServer(SFTPAbsoluteServer):
+ """A test server for sftp transports where only absolute paths will work.
+
+ It does this by serving from a deeply-nested directory that doesn't exist.
+ """
+
+ def start_server(self, backing_server=None):
+ self._server_homedir = '/dev/noone/runs/tests/here'
+ super(SFTPSiblingAbsoluteServer, self).start_server(backing_server)
+
=== modified file 'bzrlib/tests/test_selftest.py'
--- a/bzrlib/tests/test_selftest.py 2010-01-17 21:48:39 +0000
+++ b/bzrlib/tests/test_selftest.py 2010-02-03 14:08:55 +0000
@@ -62,6 +62,7 @@
)
from bzrlib.tests import (
features,
+ stub_sftp,
test_lsprof,
test_sftp_transport,
TestUtil,
@@ -1927,7 +1928,7 @@
def test_transport_sftp(self):
self.requireFeature(features.paramiko)
- self.check_transport_set(bzrlib.transport.sftp.SFTPAbsoluteServer)
+ self.check_transport_set(stub_sftp.SFTPAbsoluteServer)
def test_transport_memory(self):
self.check_transport_set(bzrlib.transport.memory.MemoryServer)
=== modified file 'bzrlib/tests/test_sftp_transport.py'
--- a/bzrlib/tests/test_sftp_transport.py 2010-01-07 03:03:01 +0000
+++ b/bzrlib/tests/test_sftp_transport.py 2010-02-03 14:08:55 +0000
@@ -46,11 +46,7 @@
if features.paramiko.available():
from bzrlib.transport import sftp as _mod_sftp
- from bzrlib.transport.sftp import (
- SFTPAbsoluteServer,
- SFTPHomeDirServer,
- SFTPTransport,
- )
+ from bzrlib.tests import stub_sftp
from bzrlib.workingtree import WorkingTree
@@ -60,9 +56,9 @@
if getattr(testcase, '_get_remote_is_absolute', None) is None:
testcase._get_remote_is_absolute = True
if testcase._get_remote_is_absolute:
- testcase.transport_server = SFTPAbsoluteServer
+ testcase.transport_server = stub_sftp.SFTPAbsoluteServer
else:
- testcase.transport_server = SFTPHomeDirServer
+ testcase.transport_server = stub_sftp.SFTPHomeDirServer
testcase.transport_readonly_server = HttpServer
@@ -162,7 +158,8 @@
self.requireFeature(features.paramiko)
def test_parse_url_with_home_dir(self):
- s = SFTPTransport('sftp://ro%62ey:h%40t@example.com:2222/~/relative')
+ s = _mod_sftp.SFTPTransport(
+ 'sftp://ro%62ey:h%40t@example.com:2222/~/relative')
self.assertEquals(s._host, 'example.com')
self.assertEquals(s._port, 2222)
self.assertEquals(s._user, 'robey')
@@ -170,7 +167,7 @@
self.assertEquals(s._path, '/~/relative/')
def test_relpath(self):
- s = SFTPTransport('sftp://user@host.com/abs/path')
+ s = _mod_sftp.SFTPTransport('sftp://user@host.com/abs/path')
self.assertRaises(errors.PathNotChild, s.relpath,
'sftp://user@host.com/~/rel/path/sub')
@@ -190,8 +187,7 @@
ssh._ssh_vendor_manager._cached_ssh_vendor = orig_vendor
def test_abspath_root_sibling_server(self):
- from bzrlib.transport.sftp import SFTPSiblingAbsoluteServer
- server = SFTPSiblingAbsoluteServer()
+ server = stub_sftp.SFTPSiblingAbsoluteServer()
server.start_server()
try:
transport = get_transport(server.get_url())
@@ -255,14 +251,13 @@
def setUp(self):
super(SSHVendorConnection, self).setUp()
- from bzrlib.transport.sftp import SFTPFullAbsoluteServer
def create_server():
"""Just a wrapper so that when created, it will set _vendor"""
# SFTPFullAbsoluteServer can handle any vendor,
# it just needs to be set between the time it is instantiated
# and the time .setUp() is called
- server = SFTPFullAbsoluteServer()
+ server = stub_sftp.SFTPFullAbsoluteServer()
server._vendor = self._test_vendor
return server
self._test_vendor = 'loopback'
@@ -415,21 +410,20 @@
self.requireFeature(features.paramiko)
def test_delay(self):
- from bzrlib.transport.sftp import SocketDelay
sending = FakeSocket()
- receiving = SocketDelay(sending, 0.1, bandwidth=1000000,
- really_sleep=False)
+ receiving = stub_sftp.SocketDelay(sending, 0.1, bandwidth=1000000,
+ really_sleep=False)
# check that simulated time is charged only per round-trip:
- t1 = SocketDelay.simulated_time
+ t1 = stub_sftp.SocketDelay.simulated_time
receiving.send("connect1")
self.assertEqual(sending.recv(1024), "connect1")
- t2 = SocketDelay.simulated_time
+ t2 = stub_sftp.SocketDelay.simulated_time
self.assertAlmostEqual(t2 - t1, 0.1)
receiving.send("connect2")
self.assertEqual(sending.recv(1024), "connect2")
sending.send("hello")
self.assertEqual(receiving.recv(1024), "hello")
- t3 = SocketDelay.simulated_time
+ t3 = stub_sftp.SocketDelay.simulated_time
self.assertAlmostEqual(t3 - t2, 0.1)
sending.send("hello")
self.assertEqual(receiving.recv(1024), "hello")
@@ -437,21 +431,20 @@
self.assertEqual(receiving.recv(1024), "hello")
sending.send("hello")
self.assertEqual(receiving.recv(1024), "hello")
- t4 = SocketDelay.simulated_time
+ t4 = stub_sftp.SocketDelay.simulated_time
self.assertAlmostEqual(t4, t3)
def test_bandwidth(self):
- from bzrlib.transport.sftp import SocketDelay
sending = FakeSocket()
- receiving = SocketDelay(sending, 0, bandwidth=8.0/(1024*1024),
- really_sleep=False)
+ receiving = stub_sftp.SocketDelay(sending, 0, bandwidth=8.0/(1024*1024),
+ really_sleep=False)
# check that simulated time is charged only per round-trip:
- t1 = SocketDelay.simulated_time
+ t1 = stub_sftp.SocketDelay.simulated_time
receiving.send("connect")
self.assertEqual(sending.recv(1024), "connect")
sending.send("a" * 100)
self.assertEqual(receiving.recv(1024), "a" * 100)
- t2 = SocketDelay.simulated_time
+ t2 = stub_sftp.SocketDelay.simulated_time
self.assertAlmostEqual(t2 - t1, 100 + 7)
=== modified file 'bzrlib/tests/test_transport.py'
--- a/bzrlib/tests/test_transport.py 2010-01-08 20:03:04 +0000
+++ b/bzrlib/tests/test_transport.py 2010-02-03 14:08:55 +0000
@@ -914,8 +914,7 @@
# SFTPFullAbsoluteServer has a get_url method, and doesn't
# override the interface (doesn't change self._vendor).
# Note that this does encryption, so can be slow.
- from bzrlib.transport.sftp import SFTPFullAbsoluteServer
- from bzrlib.tests.stub_sftp import StubServer
+ from bzrlib.tests import stub_sftp
# Start an SSH server
self.command_executed = []
@@ -924,7 +923,7 @@
# SSH channel ourselves. Surely this has already been implemented
# elsewhere?
started = []
- class StubSSHServer(StubServer):
+ class StubSSHServer(stub_sftp.StubServer):
test = self
@@ -958,7 +957,7 @@
return True
- ssh_server = SFTPFullAbsoluteServer(StubSSHServer)
+ ssh_server = stub_sftp.SFTPFullAbsoluteServer(StubSSHServer)
# We *don't* want to override the default SSH vendor: the detected one
# is the one to use.
self.start_server(ssh_server)
=== modified file 'bzrlib/transport/sftp.py'
--- a/bzrlib/transport/sftp.py 2010-01-07 03:03:01 +0000
+++ b/bzrlib/transport/sftp.py 2010-02-03 14:08:55 +0000
@@ -28,8 +28,6 @@
import itertools
import os
import random
-import select
-import socket
import stat
import sys
import time
@@ -884,332 +882,11 @@
else:
return True
-# ------------- server test implementation --------------
-import threading
-
-from bzrlib.tests.stub_sftp import StubServer, StubSFTPServer
-
-STUB_SERVER_KEY = """
------BEGIN RSA PRIVATE KEY-----
-MIICWgIBAAKBgQDTj1bqB4WmayWNPB+8jVSYpZYk80Ujvj680pOTh2bORBjbIAyz
-oWGW+GUjzKxTiiPvVmxFgx5wdsFvF03v34lEVVhMpouqPAYQ15N37K/ir5XY+9m/
-d8ufMCkjeXsQkKqFbAlQcnWMCRnOoPHS3I4vi6hmnDDeeYTSRvfLbW0fhwIBIwKB
-gBIiOqZYaoqbeD9OS9z2K9KR2atlTxGxOJPXiP4ESqP3NVScWNwyZ3NXHpyrJLa0
-EbVtzsQhLn6rF+TzXnOlcipFvjsem3iYzCpuChfGQ6SovTcOjHV9z+hnpXvQ/fon
-soVRZY65wKnF7IAoUwTmJS9opqgrN6kRgCd3DASAMd1bAkEA96SBVWFt/fJBNJ9H
-tYnBKZGw0VeHOYmVYbvMSstssn8un+pQpUm9vlG/bp7Oxd/m+b9KWEh2xPfv6zqU
-avNwHwJBANqzGZa/EpzF4J8pGti7oIAPUIDGMtfIcmqNXVMckrmzQ2vTfqtkEZsA
-4rE1IERRyiJQx6EJsz21wJmGV9WJQ5kCQQDwkS0uXqVdFzgHO6S++tjmjYcxwr3g
-H0CoFYSgbddOT6miqRskOQF3DZVkJT3kyuBgU2zKygz52ukQZMqxCb1fAkASvuTv
-qfpH87Qq5kQhNKdbbwbmd2NxlNabazPijWuphGTdW0VfJdWfklyS2Kr+iqrs/5wV
-HhathJt636Eg7oIjAkA8ht3MQ+XSl9yIJIS8gVpbPxSw5OMfw0PjVE7tBdQruiSc
-nvuQES5C9BMHjF39LZiGH1iLQy7FgdHyoP+eodI7
------END RSA PRIVATE KEY-----
-"""
-
-
-class SocketListener(threading.Thread):
-
- def __init__(self, callback):
- threading.Thread.__init__(self)
- self._callback = callback
- self._socket = socket.socket()
- self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- self._socket.bind(('localhost', 0))
- self._socket.listen(1)
- self.host, self.port = self._socket.getsockname()[:2]
- self._stop_event = threading.Event()
-
- def stop(self):
- # called from outside this thread
- self._stop_event.set()
- # use a timeout here, because if the test fails, the server thread may
- # never notice the stop_event.
- self.join(5.0)
- self._socket.close()
-
- def run(self):
- while True:
- readable, writable_unused, exception_unused = \
- select.select([self._socket], [], [], 0.1)
- if self._stop_event.isSet():
- return
- if len(readable) == 0:
- continue
- try:
- s, addr_unused = self._socket.accept()
- # because the loopback socket is inline, and transports are
- # never explicitly closed, best to launch a new thread.
- threading.Thread(target=self._callback, args=(s,)).start()
- except socket.error, x:
- sys.excepthook(*sys.exc_info())
- warning('Socket error during accept() within unit test server'
- ' thread: %r' % x)
- except Exception, x:
- # probably a failed test; unit test thread will log the
- # failure/error
- sys.excepthook(*sys.exc_info())
- warning('Exception from within unit test server thread: %r' %
- x)
-
-
-class SocketDelay(object):
- """A socket decorator to make TCP appear slower.
-
- This changes recv, send, and sendall to add a fixed latency to each python
- call if a new roundtrip is detected. That is, when a recv is called and the
- flag new_roundtrip is set, latency is charged. Every send and send_all
- sets this flag.
-
- In addition every send, sendall and recv sleeps a bit per character send to
- simulate bandwidth.
-
- Not all methods are implemented, this is deliberate as this class is not a
- replacement for the builtin sockets layer. fileno is not implemented to
- prevent the proxy being bypassed.
- """
-
- simulated_time = 0
- _proxied_arguments = dict.fromkeys([
- "close", "getpeername", "getsockname", "getsockopt", "gettimeout",
- "setblocking", "setsockopt", "settimeout", "shutdown"])
-
- def __init__(self, sock, latency, bandwidth=1.0,
- really_sleep=True):
- """
- :param bandwith: simulated bandwith (MegaBit)
- :param really_sleep: If set to false, the SocketDelay will just
- increase a counter, instead of calling time.sleep. This is useful for
- unittesting the SocketDelay.
- """
- self.sock = sock
- self.latency = latency
- self.really_sleep = really_sleep
- self.time_per_byte = 1 / (bandwidth / 8.0 * 1024 * 1024)
- self.new_roundtrip = False
-
- def sleep(self, s):
- if self.really_sleep:
- time.sleep(s)
- else:
- SocketDelay.simulated_time += s
-
- def __getattr__(self, attr):
- if attr in SocketDelay._proxied_arguments:
- return getattr(self.sock, attr)
- raise AttributeError("'SocketDelay' object has no attribute %r" %
- attr)
-
- def dup(self):
- return SocketDelay(self.sock.dup(), self.latency, self.time_per_byte,
- self._sleep)
-
- def recv(self, *args):
- data = self.sock.recv(*args)
- if data and self.new_roundtrip:
- self.new_roundtrip = False
- self.sleep(self.latency)
- self.sleep(len(data) * self.time_per_byte)
- return data
-
- def sendall(self, data, flags=0):
- if not self.new_roundtrip:
- self.new_roundtrip = True
- self.sleep(self.latency)
- self.sleep(len(data) * self.time_per_byte)
- return self.sock.sendall(data, flags)
-
- def send(self, data, flags=0):
- if not self.new_roundtrip:
- self.new_roundtrip = True
- self.sleep(self.latency)
- bytes_sent = self.sock.send(data, flags)
- self.sleep(bytes_sent * self.time_per_byte)
- return bytes_sent
-
-
-class SFTPServer(Server):
- """Common code for SFTP server facilities."""
-
- def __init__(self, server_interface=StubServer):
- self._original_vendor = None
- self._homedir = None
- self._server_homedir = None
- self._listener = None
- self._root = None
- self._vendor = ssh.ParamikoVendor()
- self._server_interface = server_interface
- # sftp server logs
- self.logs = []
- self.add_latency = 0
-
- def _get_sftp_url(self, path):
- """Calculate an sftp url to this server for path."""
- return 'sftp://foo:bar@%s:%d/%s' % (self._listener.host,
- self._listener.port, path)
-
- def log(self, message):
- """StubServer uses this to log when a new server is created."""
- self.logs.append(message)
-
- def _run_server_entry(self, sock):
- """Entry point for all implementations of _run_server.
-
- If self.add_latency is > 0.000001 then sock is given a latency adding
- decorator.
- """
- if self.add_latency > 0.000001:
- sock = SocketDelay(sock, self.add_latency)
- return self._run_server(sock)
-
- def _run_server(self, s):
- ssh_server = paramiko.Transport(s)
- key_file = pathjoin(self._homedir, 'test_rsa.key')
- f = open(key_file, 'w')
- f.write(STUB_SERVER_KEY)
- f.close()
- host_key = paramiko.RSAKey.from_private_key_file(key_file)
- ssh_server.add_server_key(host_key)
- server = self._server_interface(self)
- ssh_server.set_subsystem_handler('sftp', paramiko.SFTPServer,
- StubSFTPServer, root=self._root,
- home=self._server_homedir)
- event = threading.Event()
- ssh_server.start_server(event, server)
- event.wait(5.0)
-
- def start_server(self, backing_server=None):
- # XXX: TODO: make sftpserver back onto backing_server rather than local
- # disk.
- if not (backing_server is None or
- isinstance(backing_server, local.LocalURLServer)):
- raise AssertionError(
- "backing_server should not be %r, because this can only serve the "
- "local current working directory." % (backing_server,))
- self._original_vendor = ssh._ssh_vendor_manager._cached_ssh_vendor
- ssh._ssh_vendor_manager._cached_ssh_vendor = self._vendor
- if sys.platform == 'win32':
- # Win32 needs to use the UNICODE api
- self._homedir = getcwd()
- else:
- # But Linux SFTP servers should just deal in bytestreams
- self._homedir = os.getcwd()
- if self._server_homedir is None:
- self._server_homedir = self._homedir
- self._root = '/'
- if sys.platform == 'win32':
- self._root = ''
- self._listener = SocketListener(self._run_server_entry)
- self._listener.setDaemon(True)
- self._listener.start()
-
- def stop_server(self):
- self._listener.stop()
- ssh._ssh_vendor_manager._cached_ssh_vendor = self._original_vendor
-
- def get_bogus_url(self):
- """See bzrlib.transport.Server.get_bogus_url."""
- # this is chosen to try to prevent trouble with proxies, wierd dns, etc
- # we bind a random socket, so that we get a guaranteed unused port
- # we just never listen on that port
- s = socket.socket()
- s.bind(('localhost', 0))
- return 'sftp://%s:%s/' % s.getsockname()
-
-
-class SFTPFullAbsoluteServer(SFTPServer):
- """A test server for sftp transports, using absolute urls and ssh."""
-
- def get_url(self):
- """See bzrlib.transport.Server.get_url."""
- homedir = self._homedir
- if sys.platform != 'win32':
- # Remove the initial '/' on all platforms but win32
- homedir = homedir[1:]
- return self._get_sftp_url(urlutils.escape(homedir))
-
-
-class SFTPServerWithoutSSH(SFTPServer):
- """An SFTP server that uses a simple TCP socket pair rather than SSH."""
-
- def __init__(self):
- super(SFTPServerWithoutSSH, self).__init__()
- self._vendor = ssh.LoopbackVendor()
-
- def _run_server(self, sock):
- # Re-import these as locals, so that they're still accessible during
- # interpreter shutdown (when all module globals get set to None, leading
- # to confusing errors like "'NoneType' object has no attribute 'error'".
- class FakeChannel(object):
- def get_transport(self):
- return self
- def get_log_channel(self):
- return 'paramiko'
- def get_name(self):
- return '1'
- def get_hexdump(self):
- return False
- def close(self):
- pass
-
- server = paramiko.SFTPServer(
- FakeChannel(), 'sftp', StubServer(self), StubSFTPServer,
- root=self._root, home=self._server_homedir)
- try:
- server.start_subsystem(
- 'sftp', None, ssh.SocketAsChannelAdapter(sock))
- except socket.error, e:
- if (len(e.args) > 0) and (e.args[0] == errno.EPIPE):
- # it's okay for the client to disconnect abruptly
- # (bug in paramiko 1.6: it should absorb this exception)
- pass
- else:
- raise
- except Exception, e:
- # This typically seems to happen during interpreter shutdown, so
- # most of the useful ways to report this error are won't work.
- # Writing the exception type, and then the text of the exception,
- # seems to be the best we can do.
- import sys
- sys.stderr.write('\nEXCEPTION %r: ' % (e.__class__,))
- sys.stderr.write('%s\n\n' % (e,))
- server.finish_subsystem()
-
-
-class SFTPAbsoluteServer(SFTPServerWithoutSSH):
- """A test server for sftp transports, using absolute urls."""
-
- def get_url(self):
- """See bzrlib.transport.Server.get_url."""
- homedir = self._homedir
- if sys.platform != 'win32':
- # Remove the initial '/' on all platforms but win32
- homedir = homedir[1:]
- return self._get_sftp_url(urlutils.escape(homedir))
-
-
-class SFTPHomeDirServer(SFTPServerWithoutSSH):
- """A test server for sftp transports, using homedir relative urls."""
-
- def get_url(self):
- """See bzrlib.transport.Server.get_url."""
- return self._get_sftp_url("~/")
-
-
-class SFTPSiblingAbsoluteServer(SFTPAbsoluteServer):
- """A test server for sftp transports where only absolute paths will work.
-
- It does this by serving from a deeply-nested directory that doesn't exist.
- """
-
- def start_server(self, backing_server=None):
- self._server_homedir = '/dev/noone/runs/tests/here'
- super(SFTPSiblingAbsoluteServer, self).start_server(backing_server)
-
def get_test_permutations():
"""Return the permutations to be used in testing."""
- return [(SFTPTransport, SFTPAbsoluteServer),
- (SFTPTransport, SFTPHomeDirServer),
- (SFTPTransport, SFTPSiblingAbsoluteServer),
+ from bzrlib.tests import stub_sftp
+ return [(SFTPTransport, stub_sftp.SFTPAbsoluteServer),
+ (SFTPTransport, stub_sftp.SFTPHomeDirServer),
+ (SFTPTransport, stub_sftp.SFTPSiblingAbsoluteServer),
]
More information about the bazaar-commits
mailing list