Rev 108: Implement a PerformanceCounter module. in http://bazaar.launchpad.net/~meliae-dev/meliae/trunk
John Arbash Meinel
john at arbash-meinel.com
Tue Oct 27 14:57:42 GMT 2009
At http://bazaar.launchpad.net/~meliae-dev/meliae/trunk
------------------------------------------------------------
revno: 108
revision-id: john at arbash-meinel.com-20091027145731-rkszhleeyjncok4w
parent: john at arbash-meinel.com-20091023034828-gsk1wi0b4uqhmctw
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: trunk
timestamp: Tue 2009-10-27 09:57:31 -0500
message:
Implement a PerformanceCounter module.
This allows us to track memory consumption, etc for a child process.
We might consider rewriting all of this as a pyrex extension. It would
probably provide a bit better performance, if it is something we are
going to be calling a lot. For now, this seems good enough.
-------------- next part --------------
=== added file 'meliae/perf_counter.py'
--- a/meliae/perf_counter.py 1970-01-01 00:00:00 +0000
+++ b/meliae/perf_counter.py 2009-10-27 14:57:31 +0000
@@ -0,0 +1,163 @@
+# Copyright (C) 2009 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Get information from the OS about current memory usage"""
+
+import ctypes
+import math
+import sys
+import time
+
+
+class _Counter(object):
+ """Track various aspects of performance for a given action."""
+
+ def __init__(self, name, timer):
+ self.name = name
+ self.time_spent = 0.0
+ self._time_spent_squared = 0.0
+ self.count = 0
+ self._time_start = None
+ self._timer = timer
+
+ def tick(self):
+ """Indicate that we are starting a section related to this counter."""
+ self._time_start = self._timer()
+
+ def tock(self):
+ """Indicate that we finished processing."""
+ if self._time_start is not None:
+ now = self._timer()
+ delta = now - self._time_start
+ self.time_spent += delta
+ self._time_spent_squared += (delta * delta)
+ self._time_start = None
+ self.count += 1
+
+ def time_stddev(self):
+ # This uses a simple transformation on stddev to allow us to store 2
+ # numbers and compute the standard deviation, rather than needing a
+ # list of times.
+ # stddev = sqrt(mean(x^2) - mean(x)^2)
+ if self.count == 0:
+ return 0
+ diff = (self._time_spent_squared - (self.time_spent*self.time_spent))
+ if self.count == 1:
+ return math.sqrt(diff)
+ return math.sqrt(diff / (self.count-1))
+
+
+
+class PerformanceCounter(object):
+ """Abstract some system information about performance counters.
+
+ This includes both memory and timing.
+ """
+
+ def __init__(self):
+ self._counters = {}
+
+ def reset(self):
+ self._counters.clear()
+
+ def get_counter(self, name):
+ """Create a Counter object that will track some aspect of processing.
+
+ :param name: An identifier associated with this action.
+ :return: A Counter instance.
+ """
+ try:
+ c = self._counters[name]
+ except KeyError:
+ c = _Counter(name, self._get_timer())
+ self._counters[name] = c
+ return c
+
+ def get_memory(self, process):
+ """Ask the OS for the peak memory consumption at this point in time.
+
+ :param process: is a subprocess.Popen object.
+ :return: (current, peak) the memory used in bytes.
+ """
+ raise NotImplementedError(self.get_memory)
+
+
+class _LinuxPerformanceCounter(PerformanceCounter):
+
+ def _get_timer(self):
+ # This returns wall-clock time
+ return time.time
+
+
+class _Win32PerformanceCounter(PerformanceCounter):
+
+ def _get_timer(self):
+ # This returns wall-clock time, but using a much higher precision than
+ # time.time() [which has a resolution of only 15ms]
+ return time.clock
+
+ def _get_memory_win32(self, process_handle):
+ """Get the current memory consumption using win32 apis."""
+ mem_struct = PROCESS_MEMORY_COUNTERS_EX()
+ ret = ctypes.windll.psapi.GetProcessMemoryInfo(process_handle,
+ ctypes.byref(mem_struct),
+ ctypes.sizeof(mem_struct))
+ if not ret:
+ raise RuntimeError('Failed to call GetProcessMemoryInfo: %s'
+ % ctypes.FormatError())
+ return {
+ 'PageFaultCount': mem_struct.PageFaultCount,
+ 'PeakWorkingSetSize': mem_struct.PeakWorkingSetSize,
+ 'WorkingSetSize': mem_struct.WorkingSetSize,
+ 'QuotaPeakPagedPoolUsage': mem_struct.QuotaPeakPagedPoolUsage,
+ 'QuotaPagedPoolUsage': mem_struct.QuotaPagedPoolUsage,
+ 'QuotaPeakNonPagedPoolUsage': mem_struct.QuotaPeakNonPagedPoolUsage,
+ 'QuotaNonPagedPoolUsage': mem_struct.QuotaNonPagedPoolUsage,
+ 'PagefileUsage': mem_struct.PagefileUsage,
+ 'PeakPagefileUsage': mem_struct.PeakPagefileUsage,
+ 'PrivateUsage': mem_struct.PrivateUsage,
+ }
+
+ def get_memory(self, process):
+ """See PerformanceCounter.get_memory()"""
+ process_handle = int(process._handle)
+ mem = self._get_memory_win32(process_handle)
+ return mem['WorkingSetSize'], mem['PeakWorkingSetSize']
+
+
+# what to do about darwin, freebsd, etc?
+if sys.platform == 'win32':
+ perf_counter = _Win32PerformanceCounter()
+ # Define this here so we don't have to re-define it on every function call
+ class PROCESS_MEMORY_COUNTERS_EX(ctypes.Structure):
+ """Used by GetProcessMemoryInfo"""
+ _fields_ = [('cb', ctypes.c_ulong),
+ ('PageFaultCount', ctypes.c_ulong),
+ ('PeakWorkingSetSize', ctypes.c_size_t),
+ ('WorkingSetSize', ctypes.c_size_t),
+ ('QuotaPeakPagedPoolUsage', ctypes.c_size_t),
+ ('QuotaPagedPoolUsage', ctypes.c_size_t),
+ ('QuotaPeakNonPagedPoolUsage', ctypes.c_size_t),
+ ('QuotaNonPagedPoolUsage', ctypes.c_size_t),
+ ('PagefileUsage', ctypes.c_size_t),
+ ('PeakPagefileUsage', ctypes.c_size_t),
+ ('PrivateUsage', ctypes.c_size_t),
+ ]
+else:
+ perf_counter = _LinuxPerformanceCounter()
+
+
+def _get_process_win32():
+ """Similar to getpid() but returns an OS handle."""
+ return ctypes.windll.kernel32.GetCurrentProcess()
=== modified file 'meliae/tests/__init__.py'
--- a/meliae/tests/__init__.py 2009-09-18 17:00:34 +0000
+++ b/meliae/tests/__init__.py 2009-10-27 14:57:31 +0000
@@ -35,6 +35,7 @@
'test__loader',
'test__scanner',
'test_loader',
+ 'test_perf_counter',
'test_scanner',
]
full_names = [__name__ + '.' + n for n in module_names]
=== added file 'meliae/tests/test_perf_counter.py'
--- a/meliae/tests/test_perf_counter.py 1970-01-01 00:00:00 +0000
+++ b/meliae/tests/test_perf_counter.py 2009-10-27 14:57:31 +0000
@@ -0,0 +1,95 @@
+# Copyright (C) 2009 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import subprocess
+import sys
+
+from meliae import (
+ perf_counter,
+ tests,
+ )
+
+
+class _FakeTimer(object):
+
+ def __init__(self):
+ self._current = 0.0
+
+ def __call__(self):
+ self._current += 0.5
+ return self._current
+
+
+class Test_Counter(tests.TestCase):
+
+ def test_tick_tock(self):
+ counter = perf_counter._Counter('test', _FakeTimer())
+ counter.tick()
+ counter.tock()
+ self.assertEqual(1, counter.count)
+ self.assertEqual(0.5, counter.time_spent)
+ self.assertEqual(0.0, counter.time_stddev())
+ counter.tock()
+ self.assertEqual(2, counter.count)
+ self.assertEqual(0.5, counter.time_spent)
+
+
+
+class TestPerformanceCounter(tests.TestCase):
+
+ def setUp(self):
+ super(TestPerformanceCounter, self).setUp()
+ perf_counter.perf_counter.reset()
+
+ def tearDown(self):
+ perf_counter.perf_counter.reset()
+ super(TestPerformanceCounter, self).tearDown()
+
+ def test_perf_counter_is_not_none(self):
+ self.assertNotEqual(None, perf_counter.perf_counter)
+
+ def test_create_counter(self):
+ counter = perf_counter.perf_counter.get_counter('test-counter')
+ self.assertEqual('test-counter', counter.name)
+ self.assertEqual(counter._timer, perf_counter.perf_counter._get_timer())
+ self.assertTrue('test-counter' in perf_counter.perf_counter._counters)
+
+ def test_get_counter(self):
+ counter = perf_counter.perf_counter.get_counter('test-counter')
+ counter2 = perf_counter.perf_counter.get_counter('test-counter')
+ self.assertTrue(counter is counter2)
+
+ def test_get_memory(self):
+ # we don't have a great way to actually measure that the peak-memory
+ # value is accurate, but we can at least try
+ # Create a very large string, and then delete it.
+ p = subprocess.Popen([sys.executable, '-c',
+ 'x = "abcd"*(10*1000*1000); del x;'
+ 'import sys;'
+ 'sys.stdout.write(sys.stdin.read(3));'
+ 'sys.stdout.flush();'
+ 'sys.stdout.write(sys.stdin.read(4));'
+ 'sys.stdout.flush(); sys.stdout.close()'],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ p.stdin.write('pre')
+ p.stdout.read(3)
+ cur_mem, peak_mem = perf_counter.perf_counter.get_memory(p)
+ self.assertTrue(isinstance(cur_mem, long))
+ self.assertTrue(isinstance(peak_mem, long))
+ p.stdin.write('post')
+ p.stdout.read()
+ self.assertEqual(0, p.wait())
+ # We allocated a 40MB string, we should have peaked at at least 20MB more
+ # than we are using now.
+ self.assertTrue(peak_mem > cur_mem + 2*1000*1000)
More information about the bazaar-commits
mailing list