Hit this again

Jan 24 14:49:58 x230 kernel: [14552.596396] kvm-spice invoked oom-killer: gfp_mask=0x50, order=0, oom_score_adj=0
Jan 24 14:49:58 x230 kernel: [14552.596404] kvm-spice cpuset=emulator mems_allowed=0
Jan 24 14:49:58 x230 kernel: [14552.596409] Pid: 27542, comm: kvm-spice Tainted: GF            3.8.0-1-generic #5-Ubuntu
Jan 24 14:49:58 x230 kernel: [14552.596411] Call Trace:
Jan 24 14:49:58 x230 kernel: [14552.596422]  [<ffffffff810cfcad>] ? cpuset_print_task_mems_allowed+0x9d/0xb0
Jan 24 14:49:58 x230 kernel: [14552.596431]  [<ffffffff816bc0e8>] dump_header+0x80/0x1c1
Jan 24 14:49:58 x230 kernel: [14552.596438]  [<ffffffff81131f1e>] ? find_lock_task_mm+0x2e/0x80
Jan 24 14:49:58 x230 kernel: [14552.596444]  [<ffffffff81188f98>] ? try_get_mem_cgroup_from_mm+0x48/0x60
Jan 24 14:49:58 x230 kernel: [14552.596450]  [<ffffffff81132317>] oom_kill_process+0x1b7/0x310
Jan 24 14:49:58 x230 kernel: [14552.596454]  [<ffffffff81189090>] ? mem_cgroup_iter+0xe0/0x1e0
Jan 24 14:49:58 x230 kernel: [14552.596460]  [<ffffffff81065807>] ? has_capability_noaudit+0x17/0x20
Jan 24 14:49:58 x230 kernel: [14552.596465]  [<ffffffff8118be25>] __mem_cgroup_try_charge+0xb05/0xb50
Jan 24 14:49:58 x230 kernel: [14552.596471]  [<ffffffff8118c550>] ? mem_cgroup_charge_common+0xa0/0xa0
Jan 24 14:49:58 x230 kernel: [14552.596476]  [<ffffffff8118c501>] mem_cgroup_charge_common+0x51/0xa0
Jan 24 14:49:58 x230 kernel: [14552.596481]  [<ffffffff8118d18a>] mem_cgroup_cache_charge+0x7a/0x90
Jan 24 14:49:58 x230 kernel: [14552.596485]  [<ffffffff8112f5ea>] add_to_page_cache_locked+0x4a/0x140
Jan 24 14:49:58 x230 kernel: [14552.596490]  [<ffffffff8112f6fa>] add_to_page_cache_lru+0x1a/0x40
Jan 24 14:49:58 x230 kernel: [14552.596494]  [<ffffffff8112f7b2>] grab_cache_page_write_begin+0x92/0xf0
Jan 24 14:49:58 x230 kernel: [14552.596502]  [<ffffffff81233a21>] ext4_da_write_begin+0xb1/0x260
Jan 24 14:49:58 x230 kernel: [14552.596507]  [<ffffffff8112e5c3>] generic_file_buffered_write+0x113/0x270
Jan 24 14:49:58 x230 kernel: [14552.596512]  [<ffffffff81130284>] __generic_file_aio_write+0x1b4/0x3d0
Jan 24 14:49:58 x230 kernel: [14552.596516]  [<ffffffff811957f4>] ? __sb_start_write+0x54/0x110
Jan 24 14:49:58 x230 kernel: [14552.596520]  [<ffffffff8113051f>] generic_file_aio_write+0x7f/0x100
Jan 24 14:49:58 x230 kernel: [14552.596526]  [<ffffffff8122cf59>] ext4_file_write+0xa9/0x420
Jan 24 14:49:58 x230 kernel: [14552.596533]  [<ffffffff810816ea>] ? hrtimer_try_to_cancel+0x4a/0xd0
Jan 24 14:49:58 x230 kernel: [14552.596538]  [<ffffffff8122ceb0>] ? ext4_unwritten_wait+0xc0/0xc0
Jan 24 14:49:58 x230 kernel: [14552.596542]  [<ffffffff81193ce3>] do_sync_readv_writev+0xa3/0xe0
Jan 24 14:49:58 x230 kernel: [14552.596547]  [<ffffffff81193fb4>] do_readv_writev+0xd4/0x1e0
Jan 24 14:49:58 x230 kernel: [14552.596553]  [<ffffffff810b74e1>] ? do_futex+0x111/0x5a0
Jan 24 14:49:58 x230 kernel: [14552.596558]  [<ffffffff811940f5>] vfs_writev+0x35/0x60
Jan 24 14:49:58 x230 kernel: [14552.596562]  [<ffffffff811944a2>] sys_pwritev+0xc2/0xe0
Jan 24 14:49:58 x230 kernel: [14552.596568]  [<ffffffff816cfd1d>] system_call_fastpath+0x1a/0x1f
Jan 24 14:49:58 x230 kernel: [14552.596572] Task in /libvirt/qemu/Ubuntu-11.10-x86_64 killed as a result of limit of /libvirt/qemu/Ubuntu-11.10-x86_64
Jan 24 14:49:58 x230 kernel: [14552.596577] memory: usage 4492392kB, limit 4492392kB, failcnt 2298030
Jan 24 14:49:58 x230 kernel: [14552.596580] memory+swap: usage 0kB, limit 9007199254740991kB, failcnt 0
Jan 24 14:49:58 x230 kernel: [14552.596583] kmem: usage 0kB, limit 9007199254740991kB, failcnt 0
Jan 24 14:49:58 x230 kernel: [14552.596585] Mem-Info:
Jan 24 14:49:58 x230 kernel: [14552.596588] Node 0 DMA per-cpu:
Jan 24 14:49:58 x230 kernel: [14552.596593] CPU    0: hi:    0, btch:   1 usd:   0
Jan 24 14:49:58 x230 kernel: [14552.596596] CPU    1: hi:    0, btch:   1 usd:   0
Jan 24 14:49:58 x230 kernel: [14552.596598] CPU    2: hi:    0, btch:   1 usd:   0
Jan 24 14:49:58 x230 kernel: [14552.596600] CPU    3: hi:    0, btch:   1 usd:   0
Jan 24 14:49:58 x230 kernel: [14552.596602] Node 0 DMA32 per-cpu:
Jan 24 14:49:58 x230 kernel: [14552.596606] CPU    0: hi:  186, btch:  31 usd:  66
Jan 24 14:49:58 x230 kernel: [14552.596608] CPU    1: hi:  186, btch:  31 usd: 169
Jan 24 14:49:58 x230 kernel: [14552.596610] CPU    2: hi:  186, btch:  31 usd:  52
Jan 24 14:49:58 x230 kernel: [14552.596612] CPU    3: hi:  186, btch:  31 usd: 181
Jan 24 14:49:58 x230 kernel: [14552.596614] Node 0 Normal per-cpu:
Jan 24 14:49:58 x230 kernel: [14552.596617] CPU    0: hi:  186, btch:  31 usd: 136
Jan 24 14:49:58 x230 kernel: [14552.596619] CPU    1: hi:  186, btch:  31 usd:  60
Jan 24 14:49:58 x230 kernel: [14552.596621] CPU    2: hi:  186, btch:  31 usd: 119
Jan 24 14:49:58 x230 kernel: [14552.596623] CPU    3: hi:  186, btch:  31 usd:  54
Jan 24 14:49:58 x230 kernel: [14552.596629] active_anon:1880348 inactive_anon:521377 isolated_anon:0
Jan 24 14:49:58 x230 kernel: [14552.596629]  active_file:662160 inactive_file:727528 isolated_file:0
Jan 24 14:49:58 x230 kernel: [14552.596629]  unevictable:8103 dirty:85481 writeback:0 unstable:0
Jan 24 14:49:58 x230 kernel: [14552.596629]  free:103595 slab_reclaimable:50140 slab_unreclaimable:31299
Jan 24 14:49:58 x230 kernel: [14552.596629]  mapped:65794 shmem:120059 pagetables:13561 bounce:0
Jan 24 14:49:58 x230 kernel: [14552.596629]  free_cma:0
Jan 24 14:49:58 x230 kernel: [14552.596635] Node 0 DMA free:15900kB min:64kB low:80kB high:96kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15644kB managed:15900kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? no
Jan 24 14:49:58 x230 kernel: [14552.596643] lowmem_reserve[]: 0 3264 15839 15839
Jan 24 14:49:58 x230 kernel: [14552.596648] Node 0 DMA32 free:331720kB min:13916kB low:17392kB high:20872kB active_anon:1890552kB inactive_anon:657196kB active_file:140856kB inactive_file:228012kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:3343028kB managed:3291020kB mlocked:0kB dirty:64956kB writeback:0kB mapped:28716kB shmem:122588kB slab_reclaimable:39872kB slab_unreclaimable:12628kB kernel_stack:632kB pagetables:8996kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? no
Jan 24 14:49:58 x230 kernel: [14552.596656] lowmem_reserve[]: 0 0 12574 12574
Jan 24 14:49:58 x230 kernel: [14552.596660] Node 0 Normal free:66760kB min:53600kB low:67000kB high:80400kB active_anon:5630840kB inactive_anon:1428312kB active_file:2507784kB inactive_file:2682100kB unevictable:32412kB isolated(anon):0kB isolated(file):0kB present:12876192kB managed:12818824kB mlocked:32412kB dirty:276968kB writeback:0kB mapped:234460kB shmem:357648kB slab_reclaimable:160688kB slab_unreclaimable:112568kB kernel_stack:4192kB pagetables:45248kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? no
Jan 24 14:49:58 x230 kernel: [14552.596668] lowmem_reserve[]: 0 0 0 0
Jan 24 14:49:58 x230 kernel: [14552.596672] Node 0 DMA: 1*4kB (U) 1*8kB (U) 1*16kB (U) 0*32kB 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (R) 3*4096kB (M) = 15900kB
Jan 24 14:49:58 x230 kernel: [14552.596690] Node 0 DMA32: 128*4kB (UEM) 153*8kB (UM) 1623*16kB (UEM) 838*32kB (UEM) 339*64kB (UEM) 188*128kB (UEM) 122*256kB (UEM) 71*512kB (UEM) 74*1024kB (UEM) 41*2048kB (UM) 1*4096kB (R) = 331704kB
Jan 24 14:49:58 x230 kernel: [14552.596710] Node 0 Normal: 2061*4kB (UM) 1372*8kB (UEM) 1174*16kB (UEM) 431*32kB (UEM) 136*64kB (UEM) 17*128kB (UE) 0*256kB 0*512kB 0*1024kB 0*2048kB 1*4096kB (R) = 66772kB
Jan 24 14:49:58 x230 kernel: [14552.596727] 1516288 total pagecache pages
Jan 24 14:49:58 x230 kernel: [14552.596729] 5048 pages in swap cache
Jan 24 14:49:58 x230 kernel: [14552.596732] Swap cache stats: add 237150, delete 232102, find 148049/169464
Jan 24 14:49:58 x230 kernel: [14552.596734] Free swap  = 16425964kB
Jan 24 14:49:58 x230 kernel: [14552.596735] Total swap = 16494588kB
Jan 24 14:49:58 x230 kernel: [14552.657283] 4154864 pages RAM
Jan 24 14:49:58 x230 kernel: [14552.657288] 116655 pages reserved
Jan 24 14:49:58 x230 kernel: [14552.657291] 2151268 pages shared
Jan 24 14:49:58 x230 kernel: [14552.657294] 3024562 pages non-shared
Jan 24 14:49:58 x230 kernel: [14552.657297] [ pid ]   uid  tgid total_vm      rss nr_ptes swapents oom_score_adj name
Jan 24 14:49:58 x230 kernel: [14552.657389] [13389]   117 13389  1791539  1061964    2318     9061             0 kvm-spice
Jan 24 14:49:58 x230 kernel: [14552.657407] Memory cgroup out of memory: Kill process 28913 (kvm-spice) score 204 or sacrifice child
Jan 24 14:49:58 x230 kernel: [14552.657424] Killed process 28913 (kvm-spice) total-vm:7166156kB, anon-rss:4241340kB, file-rss:6516kB

package configuration
ii  qemu                                      1.3.0+dfsg-1~exp3ubuntu6                   amd64        fast processor emulator
ii  qemu-kvm                                  1.3.0+dfsg-1~exp3ubuntu6                   amd64        dummy transitional package from qemu-kvm to qemu-system
ii  qemu-kvm-spice                            1.2.0-2012.09-0ubuntu2                     amd64        Full virtualization on amd64 hardware
ii  qemu-system                               1.3.0+dfsg-1~exp3ubuntu6                   amd64        QEMU full system emulation binaries
ii  libvirt-bin                               1.0.1-0ubuntu1                             amd64        programs for the libvirt library
ii  libvirt0                                  1.0.1-0ubuntu1                             amd64        library for interfacing with different virtualization systems
ii  virt-manager                              0.9.4-2ubuntu2                             all          desktop application for managing virtual machines
ii  virtinst                                  0.600.3-3ubuntu1                           all          Programs to create and clone virtual machines


/sys/fs/cgroup/memory

# for f in mem*;do echo file - $f;cat $f;echo ---------------------------;done
file - memory.failcnt
0
---------------------------
file - memory.force_empty
cat: memory.force_empty: Invalid argument
---------------------------
file - memory.limit_in_bytes
9223372036854775807
---------------------------
file - memory.max_usage_in_bytes
0
---------------------------
file - memory.move_charge_at_immigrate
0
---------------------------
file - memory.numa_stat
total=2167251 N0=2167251
file=1361792 N0=1361792
anon=797356 N0=797356
unevictable=8103 N0=8103
---------------------------
file - memory.oom_control
oom_kill_disable 0
under_oom 0
---------------------------
file - memory.soft_limit_in_bytes
9223372036854775807
---------------------------
file - memory.stat
cache 6092943360
rss 2777993216
mapped_file 245870592
pgpgin 11029780
pgpgout 8864024
pgfault 15084218
pgmajfault 4512
inactive_anon 490901504
active_anon 2775019520
inactive_file 2882904064
active_file 2694995968
unevictable 33189888
hierarchical_memory_limit 9223372036854775807
total_cache 6296731648
total_rss 3945201664
total_mapped_file 245895168
total_pgpgin 13104389
total_pgpgout 10603917
total_pgfault 15978666
total_pgmajfault 5599
total_inactive_anon 947109888
total_active_anon 3486314496
total_inactive_file 3012055040
total_active_file 2769608704
total_unevictable 33189888
---------------------------
file - memory.swappiness
60
---------------------------
file - memory.usage_in_bytes
10241933312
---------------------------
file - memory.use_hierarchy
0
---------------------------



root@x230:/sys/fs/cgroup/memory/libvirt# for f in mem*;do echo file - $f;cat $f;echo ---------------------------;done
file - memory.failcnt
0
---------------------------
file - memory.force_empty
cat: memory.force_empty: Invalid argument
---------------------------
file - memory.limit_in_bytes
9223372036854775807
---------------------------
file - memory.max_usage_in_bytes
0
---------------------------
file - memory.memsw.failcnt
cat: memory.memsw.failcnt: Operation not supported
---------------------------
file - memory.memsw.limit_in_bytes
cat: memory.memsw.limit_in_bytes: Operation not supported
---------------------------
file - memory.memsw.max_usage_in_bytes
cat: memory.memsw.max_usage_in_bytes: Operation not supported
---------------------------
file - memory.memsw.usage_in_bytes
cat: memory.memsw.usage_in_bytes: Operation not supported
---------------------------
file - memory.move_charge_at_immigrate
0
---------------------------
file - memory.numa_stat
total=0 N0=0
file=0 N0=0
anon=0 N0=0
unevictable=0 N0=0
---------------------------
file - memory.oom_control
oom_kill_disable 0
under_oom 0
---------------------------
file - memory.soft_limit_in_bytes
9223372036854775807
---------------------------
file - memory.stat
cache 0
rss 0
mapped_file 0
pgpgin 0
pgpgout 0
pgfault 0
pgmajfault 0
inactive_anon 0
active_anon 0
inactive_file 0
active_file 0
unevictable 0
hierarchical_memory_limit 9223372036854775807
total_cache 0
total_rss 0
total_mapped_file 0
total_pgpgin 0
total_pgpgout 0
total_pgfault 0
total_pgmajfault 0
total_inactive_anon 0
total_active_anon 0
total_inactive_file 0
total_active_file 0
total_unevictable 0
---------------------------
file - memory.swappiness
60
---------------------------
file - memory.usage_in_bytes
0
---------------------------
file - memory.use_hierarchy
0
---------------------------
root@x230:/sys/fs/cgroup/memory/libvirt#