irqbalance overheads...
Daniel J Blueman
daniel.blueman at gmail.com
Mon Mar 29 22:12:44 UTC 2010
I'm finding on my 8-core (with Hyperthreading and no way to disable it
correctly) Core-i7 system that irqbalance does more bad than good.
It has clocked up 25 seconds in 2.5 hrs of uptime, and I see it doing
a fair amount of system work each wakeup [1], which is unfortunately
O(n) cores - it is unlikely to have saved significantly more than 25
seconds, particularly since it hasn't spread device (ie non-local)
interrupts so much [2].
Since the kernel hands out MSI vectors spread across cores, and has
choice to enable reception on the lowest-priority core, is irqbalance
really justified being installed by default, rather than say someone
who knows a particular non-MSI interrupt-heavy workload favours it?
Daniel
--- [1]
Process 4909 attached - interrupt to quit
restart_syscall(<... resuming interrupted call ...>) = 0
open("/proc/stat", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
read(3, "cpu 380611 1010827 184959 46959"..., 1024) = 1024
close(3) = 0
munmap(0x7fa757d08000, 4096) = 0
open("/proc/interrupts", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
read(3, " CPU0 CPU1 "..., 1024) = 1024
read(3, " 0 0 0 "..., 1024) = 1024
read(3, " 39: 0 0 "..., 1024) = 1024
read(3, "949 32656 32039 3"..., 1024) = 568
read(3, "", 1024) = 0
close(3) = 0
munmap(0x7fa757d08000, 4096) = 0
open("/sys/devices/system/cpu", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
getdents(3, /* 19 entries */, 32768) = 544
openat(3, "cpu0/online", O_RDONLY) = -1 ENOENT (No such file or directory)
openat(3, "cpu0/topology/core_siblings", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "ff\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu0/cache/index1/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "11\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu0/cache/index2/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "11\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu1/online", O_RDONLY) = 4
read(4, "1", 1) = 1
close(4) = 0
openat(3, "cpu1/topology/core_siblings", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "ff\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu1/cache/index1/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "22\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu1/cache/index2/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "22\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu2/online", O_RDONLY) = 4
read(4, "1", 1) = 1
close(4) = 0
openat(3, "cpu2/topology/core_siblings", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "ff\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu2/cache/index1/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "44\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu2/cache/index2/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "44\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu3/online", O_RDONLY) = 4
read(4, "1", 1) = 1
close(4) = 0
openat(3, "cpu3/topology/core_siblings", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "ff\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu3/cache/index1/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "88\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu3/cache/index2/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "88\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu4/online", O_RDONLY) = 4
read(4, "1", 1) = 1
close(4) = 0
openat(3, "cpu4/topology/core_siblings", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "ff\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu4/cache/index1/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "11\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu4/cache/index2/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "11\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu5/online", O_RDONLY) = 4
read(4, "1", 1) = 1
close(4) = 0
openat(3, "cpu5/topology/core_siblings", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "ff\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu5/cache/index1/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "22\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu5/cache/index2/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "22\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu6/online", O_RDONLY) = 4
read(4, "1", 1) = 1
close(4) = 0
openat(3, "cpu6/topology/core_siblings", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "ff\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu6/cache/index1/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "44\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu6/cache/index2/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "44\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu7/online", O_RDONLY) = 4
read(4, "1", 1) = 1
close(4) = 0
openat(3, "cpu7/topology/core_siblings", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "ff\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu7/cache/index1/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "88\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
openat(3, "cpu7/cache/index2/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR) = 0
read(4, "88\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d08000, 4096) = 0
getdents(3, /* 0 entries */, 32768) = 0
close(3) = 0
open("/proc/net/dev", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
read(3, "Inter-| Receive "..., 1024) = 578
socket(PF_INET, SOCK_DGRAM, IPPROTO_IP) = 4
ioctl(4, SIOCETHTOOL, 0x7fff05bc7e30) = -1 EOPNOTSUPP (Operation not
supported)
close(4) = 0
socket(PF_INET, SOCK_DGRAM, IPPROTO_IP) = 4
ioctl(4, SIOCETHTOOL, 0x7fff05bc7e30) = 0
close(4) = 0
open("/sys/bus/pci/devices/0000:0b:00.0/irq", O_RDONLY) = 4
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d07000
read(4, "36\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d07000, 4096) = 0
socket(PF_INET, SOCK_DGRAM, IPPROTO_IP) = 4
ioctl(4, SIOCETHTOOL, 0x7fff05bc7e30) = 0
close(4) = 0
open("/sys/bus/pci/devices/0000:05:00.0/irq", O_RDONLY) = 4
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d07000
read(4, "39\n", 4096) = 3
close(4) = 0
munmap(0x7fa757d07000, 4096) = 0
read(3, "", 1024) = 0
close(3) = 0
munmap(0x7fa757d08000, 4096) = 0
nanosleep({9, 657576000}, ^C <unfinished ...>
Process 4909 detached
--- [2]
# cat /proc/interrupts
CPU0 CPU1 CPU2 CPU3 CPU4
CPU5 CPU6 CPU7
0: 31 0 0 492865 0
0 0 0 IO-APIC-edge timer
1: 1 0 0 12079 0
0 0 0 IO-APIC-edge i8042
8: 0 0 0 1 0
0 0 0 IO-APIC-edge rtc0
9: 0 0 0 61 0
4 319 0 IO-APIC-fasteoi acpi
12: 1 0 0 154174 0
808315 0 0 IO-APIC-edge i8042
16: 0 0 0 0 0
0 62 0 IO-APIC-fasteoi ehci_hcd:usb1, mmc0
23: 0 0 0 0 45
0 0 0 IO-APIC-fasteoi ehci_hcd:usb2
24: 773788 0 0 0 0
0 0 0 HPET_MSI-edge hpet2
25: 0 515679 0 0 0
0 0 0 HPET_MSI-edge hpet3
26: 0 0 465773 0 0
0 0 0 HPET_MSI-edge hpet4
27: 0 0 0 425523 0
0 0 0 HPET_MSI-edge hpet5
28: 0 0 0 0 578571
0 0 0 HPET_MSI-edge hpet6
35: 1 0 0 4550 0
63602 0 0 PCI-MSI-edge ahci
36: 0 0 0 0 9940164
0 0 0 PCI-MSI-edge eth0
37: 0 0 0 0 0
16045 75 0 PCI-MSI-edge hda_intel
38: 0 0 0 0 0
0 76 0 PCI-MSI-edge hda_intel
39: 0 0 0 0 0
0 0 0 PCI-MSI-edge iwlagn
NMI: 91935 222671 57154 97747 58357
199249 50607 56186 Non-maskable interrupts
LOC: 190 175 155 135 115
378385 351065 370748 Local timer interrupts
SPU: 0 0 0 0 0
0 0 0 Spurious interrupts
PMI: 0 0 0 0 0
0 0 0 Performance monitoring interrupts
PND: 0 0 0 0 0
0 0 0 Performance pending work
RES: 56646 32751 28175 26262 43184
22535 23754 22636 Rescheduling interrupts
CAL: 3013 3024 2555 3114 2668
2651 2440 2670 Function call interrupts
TLB: 122387 32478 36100 28363 47961
32659 32043 35694 TLB shootdowns
TRM: 3018 3004 3006 3009 3018
3004 3006 3009 Thermal event interrupts
THR: 0 0 0 0 0
0 0 0 Threshold APIC interrupts
MCE: 0 0 0 0 0
0 0 0 Machine check exceptions
MCP: 36 33 33 34 34
33 33 33 Machine check polls
ERR: 7
MIS: 0
--
Daniel J Blueman
More information about the kernel-team
mailing list