Rev 18: Complete decorator, report bytes written. in file:///v/home/vila/.bazaar/plugins/transportstats/
Vincent Ladeuil
v.ladeuil+lp at free.fr
Mon Oct 8 15:19:15 BST 2007
At file:///v/home/vila/.bazaar/plugins/transportstats/
------------------------------------------------------------
revno: 18
revision-id: v.ladeuil+lp at free.fr-20071008141914-flsiq3tt1xw6lyux
parent: v.ladeuil+lp at free.fr-20071008121619-hmmlpdvlh3v8qy7w
committer: Vincent Ladeuil <v.ladeuil+lp at free.fr>
branch nick: transportstats
timestamp: Mon 2007-10-08 16:19:14 +0200
message:
Complete decorator, report bytes written.
* tests/blackbox.py:
(TestWithStatsFile.run_bzr_with_stats_collection): Nasty test
suite side-effect. If a test has left the stats file opened, we
don't know where anymore.
(TestTsDisplay.test_display): Test all stats displayed.
* statsfile.py:
File not saved in previous commit, damn DVC. Was cosmetic
fortunately.
* stats.py:
Register new stats.
(Stats.collect): Simplified. Add bytes written collection.
* decorator.py:
(StatsCollector.append_file, StatsCollector.append_bytes,
StatsCollector.delete, StatsCollector.delete_tree,
StatsCollector.mkdir, StatsCollector.put_file,
StatsCollector.put_bytes, StatsCollector.rename,
StatsCollector.rmdir, StatsCollector.lock_read,
StatsCollector.lock_write): Add missing methods.
(StatsCollector.readv): Use args and kwargs to make it more
robust. Should it be generalized ?
* commands.py:
(cmd_ts_display.run): Add bytes written.
modified:
commands.py commands.py-20070927123419-zy82qayy9xzwxvzi-1
decorator.py decorator.py-20070926152401-52kuiex1mu755ajk-1
stats.py stats.py-20070928061304-7i3r2h4gg6rbi03e-1
statsfile.py structuredfile.py-20070927123433-krruwbx4mkalu3xs-1
tests/blackbox.py blackbox.py-20070930083839-9pz5gsxwefw4i294-1
-------------- next part --------------
=== modified file 'commands.py'
--- a/commands.py 2007-10-08 09:46:16 +0000
+++ b/commands.py 2007-10-08 14:19:14 +0000
@@ -57,9 +57,10 @@
from bzrlib.plugins.transportstats import stats
mystats = stats.get_stats()
mystats.start_exploiting()
- (requests, bytes_read, avg_latency) = mystats.collect()
+ (requests, bytes_read, bytes_written, avg_latency) = mystats.collect()
print '# requests: %d' % requests
print 'Bytes read: %d' % bytes_read
+ print 'Bytes written: %d' % bytes_read
# FIXME: It may be more appropriate to divide by 2 as ping does.
print 'Average latency: %dms' % avg_latency
mystats.stop_exploiting()
=== modified file 'decorator.py'
--- a/decorator.py 2007-10-08 12:16:19 +0000
+++ b/decorator.py 2007-10-08 14:19:14 +0000
@@ -63,16 +63,60 @@
latency = int((time.time() - self.__start_time) * 1000)
return max(0, min(65535, latency))
+ def __pump_to_tmp(self, f):
+ """Copy the file-like object content to a temp file.
+
+ :returns: temp_file, nb_bytes. temp_file is the temporary file
+ handle. Should be closed by caller after use. nb_bytes is the number
+ of bytes copied.
+ """
+ temp_file = tempfile.TemporaryFile()
+ nb_bytes = self._pump(f, temp_file)
+ temp_file.seek(0)
+ return temp_file, nb_bytes
+
@classmethod
def _get_url_prefix(self):
"""Stats transport decorators are invoked via 'stats+'"""
return 'stats+'
- def has(self, relpath):
- """See Transport.has()."""
- self.__start()
- ret = self._decorated.has(relpath)
- self.__collect('Transport.has', relpath, self.__latency())
+ def append_file(self, relpath, f, mode=None):
+ """See Transport.append_file()."""
+ saved_pos = f.tell()
+ temp_file, nb_bytes = self.__pump_to_tmp(f)
+ self.__start()
+ try:
+ ret = self._decorated.append_file(relpath, temp_file, mode=mode)
+ except:
+ f.seek(saved_pos)
+ temp_file.close()
+ raise
+ else:
+ self.__collect('Transport.append_file', relpath, nb_bytes,
+ self.__latency())
+ temp_file.close()
+ return ret
+
+ def append_bytes(self, relpath, bytes, mode=None):
+ """See Transport.append_bytes()."""
+ self.__start()
+ ret = self._decorated.append_bytes(relpath, bytes, mode=mode)
+ self.__collect('Transport.append_bytes', relpath, len(bytes),
+ self.__latency())
+ return ret
+
+ def delete(self, relpath):
+ """See Transport.delete()."""
+ self.__start()
+ ret = self._decorated.delete(relpath)
+ self.__collect('Transport.delete', relpath, self.__latency())
+ return ret
+
+ def delete_tree(self, relpath):
+ """See Transport.delete_tree()."""
+ self.__start()
+ ret = self._decorated.delete_tree(relpath)
+ self.__collect('Transport.delete_tree', relpath, self.__latency())
return ret
def get(self, relpath):
@@ -80,6 +124,7 @@
self.__start()
f = self._decorated.get(relpath)
latency = self.__latency()
+
temp_file = tempfile.TemporaryFile()
nb_bytes = self._pump(f, temp_file)
temp_file.seek(0)
@@ -97,7 +142,48 @@
self.__latency())
return bytes
- def readv(self, relpath, offsets):
+ def has(self, relpath):
+ """See Transport.has()."""
+ self.__start()
+ ret = self._decorated.has(relpath)
+ self.__collect('Transport.has', relpath, self.__latency())
+ return ret
+
+ def mkdir(self, relpath, mode=None):
+ """See Transport.mkdir()."""
+ self.__start()
+ ret = self._decorated.mkdir(relpath, mode)
+ self.__collect('Transport.mkdir', relpath, self.__latency())
+ return ret
+
+ #FIXME:open_write_stream may need several implementations
+
+ def put_file(self, relpath, f, mode=None):
+ """See Transport.put_file()."""
+ saved_pos = f.tell()
+ temp_file, nb_bytes = self.__pump_to_tmp(f)
+ self.__start()
+ try:
+ ret = self._decorated.put_file(relpath, temp_file, mode)
+ except:
+ f.seek(saved_pos)
+ temp_file.close()
+ raise
+ else:
+ self.__collect('Transport.put_file', relpath, nb_bytes,
+ self.__latency())
+ temp_file.close()
+ return ret
+
+ def put_bytes(self, relpath, bytes, mode=None):
+ """See Transport.put_bytes()."""
+ self.__start()
+ ret = self._decorated.put_bytes(relpath, bytes, mode)
+ self.__collect('Transport.put_bytes', relpath, len(bytes),
+ self.__latency())
+ return ret
+
+ def readv(self, relpath, offsets, *args, **kwargs):
"""See Transport.readv()."""
self.__start()
first = True
@@ -108,13 +194,27 @@
# default Transport implementation used by ftp, local and sftp; 2 -
# http; 3 - remote). For now we collect only data "visible" at this API
# level which provides a rough estimate (1 transaction, n bytes read).
- for (pos, data) in self._decorated.readv(relpath, offsets):
+ for (pos, data) in self._decorated.readv(relpath, offsets,
+ *args, **kwargs):
if first:
self.__collect('Transport.readv', relpath, self.__latency())
first = False
self.__collect('Transport.readv/offset', relpath, pos, len(data))
yield pos, data
+ def rename(self, rel_from, rel_to):
+ self.__start()
+ ret = self._decorated.rename(rel_from, rel_to)
+ self.__collect('Transport.rename', rel_from, rel_to, self.__latency())
+ return ret
+
+ def rmdir(self, relpath):
+ """See Transport.rmdir."""
+ self.__start()
+ ret = self._decorated.rmdir(relpath)
+ self.__collect('Transport.rmdir', relpath, self.__latency())
+ return ret
+
def stat(self, relpath):
"""See Transport.stat()."""
self.__start()
@@ -122,6 +222,19 @@
self.__collect('Transport.stat', relpath, self.__latency())
return ret
+ def lock_read(self, relpath):
+ """See Transport.lock_read."""
+ self.__start()
+ ret = self._decorated.lock_read(relpath)
+ self.__collect('Transport.lock_read', relpath, self.__latency())
+ return ret
+
+ def lock_write(self, relpath):
+ """See Transport.lock_write."""
+ self.__start()
+ ret = self._decorated.lock_write(relpath)
+ self.__collect('Transport.lock_write', relpath, self.__latency())
+ return ret
class StatsServer(decorator.DecoratorServer):
=== modified file 'stats.py'
--- a/stats.py 2007-10-08 12:16:19 +0000
+++ b/stats.py 2007-10-08 14:19:14 +0000
@@ -97,18 +97,40 @@
# Shortcut
register_stat = stats_registry.register
-register_stat('Transport.has', TransportStat,
+register_stat('Transport.append_file', TransportStat,
+ '%(base)us%(relpath)us%(bytes_written)L%(latency)H')
+register_stat('Transport.append_bytes', TransportStat,
+ '%(base)us%(relpath)us%(bytes_written)L%(latency)H')
+register_stat('Transport.delete', TransportStat,
+ '%(base)us%(relpath)us%(latency)H')
+register_stat('Transport.delete_tree', TransportStat,
'%(base)us%(relpath)us%(latency)H')
register_stat('Transport.get', TransportStat,
'%(base)us%(relpath)us%(bytes_read)L%(latency)H')
register_stat('Transport.get_bytes', TransportStat,
'%(base)us%(relpath)us%(bytes_read)L%(latency)H')
+register_stat('Transport.has', TransportStat,
+ '%(base)us%(relpath)us%(latency)H')
+register_stat('Transport.mkdir', TransportStat,
+ '%(base)us%(relpath)us%(latency)H')
+register_stat('Transport.put_file', TransportStat,
+ '%(base)us%(relpath)us%(bytes_written)L%(latency)H')
+register_stat('Transport.put_bytes', TransportStat,
+ '%(base)us%(relpath)us%(bytes_written)L%(latency)H')
register_stat('Transport.readv', TransportStat,
'%(base)us%(relpath)us%(latency)H')
register_stat('Transport.readv/offset', TransportReadvOffset,
'%(base)us%(relpath)us%(offset)L%(bytes_read)L')
+register_stat('Transport.rename', TransportStat,
+ '%(base)us%(relpath)us%(target)us%(latency)H')
+register_stat('Transport.rmdir', TransportStat,
+ '%(base)us%(relpath)us%(latency)H')
register_stat('Transport.stat', TransportStat,
'%(base)us%(relpath)us%(latency)H')
+register_stat('Transport.lock_read', TransportStat,
+ '%(base)us%(relpath)us%(latency)H')
+register_stat('Transport.lock_write', TransportStat,
+ '%(base)us%(relpath)us%(latency)H')
class Stats(object):
@@ -214,23 +236,21 @@
def collect(self):
requests = 0
bytes_read = 0
+ bytes_written = 0
latency_count = 0
total_latency = 0
for stat in self:
- try:
- bytes_read += stat.bytes_read
- except:
- pass
- try:
- total_latency += stat.latency
+ bytes_read += getattr(stat, 'bytes_read', 0)
+ bytes_written += getattr(stat, 'bytes_written', 0)
+ latency = getattr(stat, 'latency', None)
+ if latency is not None:
+ total_latency += latency
latency_count += 1
- except:
- pass
- requests += stat.requests
+ requests += getattr(stat, 'requests', 0)
avg_latency = 0
if latency_count:
avg_latency = total_latency / latency_count
- return requests, bytes_read, avg_latency
+ return requests, bytes_read, bytes_written, avg_latency
_stats = None
=== modified file 'statsfile.py'
--- a/statsfile.py 2007-10-08 12:16:19 +0000
+++ b/statsfile.py 2007-10-08 14:19:14 +0000
@@ -348,11 +348,13 @@
self.read_header('marker')
def read_stat_name(self, raw_stat, name_fmt):
+ # FIXME: layer violation between Stats/StatsFile
file = StringIO(raw_stat)
name = name_fmt.unpack(file)
return name, file
def read_stat_body(self, f, body_fmt):
+ # FIXME: layer violation between Stats/StatsFile
body_fmt.set_unique_strings_container(self.unique_strings)
return body_fmt.unpack(f)
=== modified file 'tests/blackbox.py'
--- a/tests/blackbox.py 2007-10-08 09:46:16 +0000
+++ b/tests/blackbox.py 2007-10-08 14:19:14 +0000
@@ -39,6 +39,9 @@
# exploitation. We mix things up in the test suite, so we need to
# explictely control the collection
st = stats.get_stats()
+ # If some other tests have been run, they may have left the stats file
+ # opened for collecting, be sure to start in a clean state.
+ st._force_stop_collecting()
st.start_collecting()
self.run_bzr(*args, **kwargs)
# Each decorator use triggers a start_collecting, we have to close for
@@ -62,7 +65,10 @@
def test_display(self):
self._decorated_bzr_branch()
out = self.run_bzr('ts-display')[0]
+ self.assertContainsRe(out, '# requests: ')
self.assertContainsRe(out, 'Bytes read: ')
+ self.assertContainsRe(out, 'Bytes written: ')
+ self.assertContainsRe(out, 'Average latency: ')
class TestTsReset(tests.TestCase):
More information about the bazaar-commits
mailing list