doc/Documentation/vm/hugetlbpage.txt に
Examples
========1) map_hugetlb: see tools/testing/selftests/vm/map_hugetlb.c
2) hugepage-shm: see tools/testing/selftests/vm/hugepage-shm.c
3) hugepage-mmap: see tools/testing/selftests/vm/hugepage-mmap.c
4) The libhugetlbfs (http://libhugetlbfs.sourceforge.net) library provides a
https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt
wide range of userspace tools to help with huge page usability, environment
setup, and control. Furthermore it provides useful test cases that should be
used when modifying code to ensure no regressions are introduced.
と書かれていたので、hugepage-shm.c をダウンロードしてコンパイルして実行してみた。
% wget https://raw.github.com/mirrors/linux-2.6/master/tools/testing/selftests/vm/hugepage-shm.c % gcc -g -o hugepage-shm hugepage-shm.c % su # sysctl -w vm.nr_hugepages=200 vm.nr_hugepages = 200 # ./hugepage-shm shmid: 0xb78025 shmaddr: 0x7f9319800000 Starting the writes: ................................................................................................................................................................................................................................................................ Starting the Check...Done.
hugepage-shm を gdb でステップ実行しながら、
# cat /proc/meminfo |grep -i huge AnonHugePages: 0 kB HugePages_Total: 200 HugePages_Free: 200 HugePages_Rsvd: 0 HugePages_Surp: 0 Hugepagesize: 2048 kB # gdb ./hugepage-shm NU gdb (GDB) Red Hat Enterprise Linux (7.2-60.el6) Copyright (C) 2010 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-redhat-linux-gnu". For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>... Reading symbols from /home/yazekats/work/hugepage_test/hugepage-shm...done. (gdb) (gdb) b main Breakpoint 1 at 0x40070c: file hugepage-shm.c, line 60. (gdb) run Starting program: /home/yazekats/work/hugepage_test/hugepage-shm Breakpoint 1, main () at hugepage-shm.c:60 60 shmid = shmget(2, LENGTH, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); Missing separate debuginfos, use: debuginfo-install glibc-2.12-1.132.el6.x86_64 (gdb) n 61 if (shmid < 0) { (gdb) n 65 printf("shmid: 0x%x\n", shmid); (gdb) n shmid: 0xd1802d 67 shmaddr = shmat(shmid, ADDR, SHMAT_FLAGS); (gdb) n 68 if (shmaddr == (char *)-1) { (gdb) n 73 printf("shmaddr: %p\n", shmaddr); (gdb) n shmaddr: 0x2aaaaac00000 75 dprintf("Starting the writes:\n"); (gdb) n Starting the writes: 76 for (i = 0; i < LENGTH; i++) { (gdb) n 77 shmaddr[i] = (char)(i); (gdb) n 78 if (!(i % (1024 * 1024)))
/proc/meminfo を見てみると、
% cat /proc/meminfo|grep -i huge AnonHugePages: 0 kB HugePages_Total: 200 HugePages_Free: 199 HugePages_Rsvd: 127 HugePages_Surp: 0 Hugepagesize: 2048 kB
HugePages_Free が減って、HugePages_Rsvd が使われ、HugePage が使われている様子が見れる。
補足
vm.nr_hugepages が足りないと "shmget: Cannot allocate memory" と怒られる。
[root@yazekats-linux hugepage_test]# sysctl -w vm.nr_hugepages=100 vm.nr_hugepages = 100 [root@yazekats-linux hugepage_test]# ./hugepage-shm shmget: Cannot allocate memory
strace でシステムコールのトレースを調べると shmget の戻り値が "-1 ENOMEM (Cannot allocate memory)" となっている。
[root@yazekats-linux hugepage_test]# strace ./hugepage-shm ... shmget(0x2, 268435456, IPC_CREAT|SHM_HUGETLB|0600) = -1 ENOMEM (Cannot allocate memory) dup(2) = 3 fcntl(3, F_GETFL) = 0x8002 (flags O_RDWR|O_LARGEFILE) brk(0) = 0x1fdf000 brk(0x2000000) = 0x2000000 fstat(3, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 0), ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f9bfc7f4000 lseek(3, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek) write(3, "shmget: Cannot allocate memory\n", 31shmget: Cannot allocate memory ) = 31 close(3) = 0 munmap(0x7f9bfc7f4000, 4096) = 0 exit_group(1) = ?
追記:
ipcs コマンドで共有メモリが確保される様子を追ってみた。
まず実行前の状況を確認して、
# cat /proc/meminfo|grep -i huge AnonHugePages: 0 kB HugePages_Total: 200 HugePages_Free: 200 HugePages_Rsvd: 0 HugePages_Surp: 0 Hugepagesize: 2048 kB # ipcs -um ------ Shared Memory Status -------- segments allocated 45 pages allocated 3930 pages resident 3188 pages swapped 0 Swap performance: 0 attempts 0 successes
hugepage-shm を実行して、
# gdb ./hugepage-shm GNU gdb (GDB) Red Hat Enterprise Linux (7.2-60.el6) Copyright (C) 2010 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-redhat-linux-gnu". For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>... Reading symbols from /home/yazekats/work/hugepage_test/hugepage-shm...done. (gdb) b main Breakpoint 1 at 0x40070c: file hugepage-shm.c, line 60. (gdb) run Starting program: /home/yazekats/work/hugepage_test/hugepage-shm Breakpoint 1, main () at hugepage-shm.c:60 60 shmid = shmget(2, LENGTH, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); Missing separate debuginfos, use: debuginfo-install glibc-2.12-1.132.el6.x86_64 (gdb) n 61 if (shmid < 0) { (gdb) n 65 printf("shmid: 0x%x\n", shmid); (gdb) n shmid: 0x1078026 67 shmaddr = shmat(shmid, ADDR, SHMAT_FLAGS); (gdb) n 68 if (shmaddr == (char *)-1) { (gdb) n 73 printf("shmaddr: %p\n", shmaddr); (gdb) n shmaddr: 0x2aaaaac00000 75 dprintf("Starting the writes:\n"); (gdb)
実行中の状況を確認すると、
# cat /proc/meminfo|grep -i huge AnonHugePages: 0 kB HugePages_Total: 200 HugePages_Free: 200 HugePages_Rsvd: 128 <-- 増えている HugePages_Surp: 0 Hugepagesize: 2048 kB # ipcs -um ------ Shared Memory Status -------- segments allocated 46 <-- 増えている pages allocated 69466 <-- 増えている pages resident 3188 pages swapped 0 Swap performance: 0 attempts 0 successes # ipcs -a|grep root 0x00000002 17268774 root 600 268435456 1 <-- たぶんこれ ... # ipcs -m -i 17268774 Shared memory Segment shmid=17268774 uid=0 gid=0 cuid=0 cgid=0 mode=0600 access_perms=0600 bytes=268435456 lpid=5296 cpid=5296 nattch=1 att_time=Sun Feb 23 22:58:37 2014 det_time=Not set change_time=Sun Feb 23 22:58:34 2014
さらに追記:
pmap も見てみた。
# pmap -x 5374 5374: /home/yazekats/work/hugepage_test/hugepage-shm Address Kbytes RSS Dirty Mode Mapping 0000000000400000 4 4 4 r-x-- hugepage-shm 0000000000600000 4 4 4 rw--- hugepage-shm 0000003ba6200000 128 108 12 r-x-- ld-2.12.so 0000003ba641f000 4 4 4 r---- ld-2.12.so 0000003ba6420000 4 4 4 rw--- ld-2.12.so 0000003ba6421000 4 4 4 rw--- [ anon ] 0000003ba6600000 1580 224 4 r-x-- libc-2.12.so 0000003ba678b000 2044 0 0 ----- libc-2.12.so 0000003ba698a000 16 8 8 r---- libc-2.12.so 0000003ba698e000 4 4 4 rw--- libc-2.12.so 0000003ba698f000 20 12 12 rw--- [ anon ] 00002aaaaac00000 262144 0 0 rw-s- SYSV00000002 (deleted) <-- これ 00007ffff7fdf000 12 12 12 rw--- [ anon ] 00007ffff7ffc000 8 8 8 rw--- [ anon ] 00007ffff7ffe000 4 4 0 r-x-- [ anon ] 00007ffffffde000 132 8 8 rw--- [ stack ] ffffffffff600000 4 0 0 r-x-- [ anon ] ---------------- ------ ------ ------ total kB 266116 408 88
HugePage は最初はカーネル空間に確保されて、使われるとユーザー空間に移動するのかな？
