devmem读写物理内存和devkmem读取内核虚拟内存

关键词：/dev/mem、/dev/kmem、mmap、__va、__pa、remap_pfn_range等等。

在日常工作中常有直接操作寄存器或者某一物理地址的需求，busybox中提供了devmem。通过它可以读写物理内存。

它的实现借助mmap和/dev/mem，通过mmap将/dev/mem物理地址映射到用户空间，devmem就可以像操作虚拟地址一样进行读写。

hexdump同样也可以类似devmem的功能。

如果需要在用户空间获取内核某个变量值，可以使用devkmem通过/dev/kmem进行。

下面分别介绍这三种工具。

1. devmem操作物理地址，它是如何做到的？

用户空间是无法直接操作物理地址的；但是日常工作中常需要对某一物理地址进行读写，尤其是寄存器。

devmem可以实现这个功能。那么devmem做了什么？/dev/mem在内核中优势如何实现的呢？

1.1 devmem工具使用

devmem使用介绍如下：

BusyBox v1.27.2 (-- :: CST) multi-call binary.

Usage: devmem ADDRESS [WIDTH [VALUE]]

Read/write from physical address

        ADDRESS Address to act upon
        WIDTH   Width (//...)
        VALUE   Data to be written

devmem的能力有限，只能处理最大64字节的数目。

下面向0xfc20700这个地址写入32位数据0x12345678：

devmem 0xfc20700  0x12345678

然后从0xfc20700读取进行验证。

devmem 0xfc20700
0x12345678

1.2 devmem工具分析

从下面的代码可知，devmem解析参数，然后将地址转换成页面对齐的地址。mmap将/dev/mem的输入地址偏移的页面映射到用户空间，然后读取数值。

int devmem_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
int devmem_main(int argc UNUSED_PARAM, char **argv)
{
    void *map_base, *virt_addr;
    uint64_t read_result;
    uint64_t writeval = writeval; /* for compiler */
    off_t target;
    unsigned page_size, mapped_size, offset_in_page;
    int fd;
    unsigned width =  * sizeof(int);

    /* ADDRESS */
    if (!argv[])
        bb_show_usage();
    errno = ;
    target = bb_strtoull(argv[], NULL, ); /* allows hex, oct etc */---------------第一个参数是地址

    /* WIDTH */
    if (argv[]) {------------------------------------------------------------------第二个参数，在写的情况下，需要知道写数据的位宽。
        if (isdigit(argv[][]) || argv[][])
            width = xatou(argv[]);
        else {
            static const char bhwl[] ALIGN1 = "bhwl";
            static const uint8_t sizes[] ALIGN1 = {
                 * sizeof(char),
                 * sizeof(short),
                 * sizeof(int),
                 * sizeof(long),
                 /* bad */
            };
            width = strchrnul(bhwl, (argv[][] | 0x20)) - bhwl;
            width = sizes[width];
        }
        /* VALUE */
        if (argv[])-----------------------------------------------------------------第三个参数，待写入数值。
            writeval = bb_strtoull(argv[], NULL, );
    } else { /* argv[2] == NULL */
        /* make argv[3] to be a valid thing to fetch */
        argv--;
    }
    if (errno)
        bb_show_usage(); /* one of bb_strtouXX failed */

    fd = xopen("/dev/mem", argv[] ? (O_RDWR | O_SYNC) : (O_RDONLY | O_SYNC));-------根据第三个参数确定是以只读形式打开，还是以读写形式打开。/dev/mem代表整个内核空间。
    mapped_size = page_size = getpagesize();
    offset_in_page = (unsigned)target & (page_size - );-----------------------------对地址进行也对齐。
    if (offset_in_page + width > page_size) {----------------------------------------如果跨页，则mapped_size编程两个页面。
        /* This access spans pages.
         * Must map two pages to make it possible: */
        mapped_size *= ;
    }
    map_base = mmap(NULL,
            mapped_size,
            argv[] ? (PROT_READ | PROT_WRITE) : PROT_READ,
            MAP_SHARED,
            fd,
            target & ~(off_t)(page_size - ));---------------------------------------将/dev/mem文件的从target的页对齐偏移开始，映射mapped_size块大小内存。映射结果是map_base。
    if (map_base == MAP_FAILED)
        bb_perror_msg_and_die("mmap");

//    printf("Memory mapped at address %p.\n", map_base);

    virt_addr = (char*)map_base + offset_in_page;

    if (!argv[]) {
        switch (width) {
        case :
            read_result = *(volatile uint8_t*)virt_addr;
            break;
        case :
            read_result = *(volatile uint16_t*)virt_addr;
            break;
        case :
            read_result = *(volatile uint32_t*)virt_addr;
            break;
        case :
            read_result = *(volatile uint64_t*)virt_addr;
            break;
        default:
            bb_error_msg_and_die("bad width");
        }
//        printf("Value at address 0x%"OFF_FMT"X (%p): 0x%llX\n",
//            target, virt_addr,
//            (unsigned long long)read_result);
        /* Zero-padded output shows the width of access just done */
        printf("0x%0*llX\n", (width >> ), (unsigned long long)read_result);------------读取数据并打印。
    } else {
        switch (width) {
        case :
            *(volatile uint8_t*)virt_addr = writeval;
//            read_result = *(volatile uint8_t*)virt_addr;
            break;
        case :
            *(volatile uint16_t*)virt_addr = writeval;
//            read_result = *(volatile uint16_t*)virt_addr;
            break;
        case :
            *(volatile uint32_t*)virt_addr = writeval;
//            read_result = *(volatile uint32_t*)virt_addr;
            break;
        case :
            *(volatile uint64_t*)virt_addr = writeval;
//            read_result = *(volatile uint64_t*)virt_addr;
            break;
        default:
            bb_error_msg_and_die("bad width");
        }
//        printf("Written 0x%llX; readback 0x%llX\n",
//                (unsigned long long)writeval,
//                (unsigned long long)read_result);
    }

    if (ENABLE_FEATURE_CLEAN_UP) {
        if (munmap(map_base, mapped_size) == -)
            bb_perror_msg_and_die("munmap");
        close(fd);
    }

    return EXIT_SUCCESS;
}

1.3 /dev/mem是如何实现物理地址读写？

/dev/mem在chr_dev_init()中创建，需要创建的节点在devlist[]中。

static const struct memdev {
    const char *name;
    umode_t mode;
    const struct file_operations *fops;
    fmode_t fmode;
} devlist[] = {
#ifdef CONFIG_DEVMEM
     [] = { "mem", , &mem_fops, FMODE_UNSIGNED_OFFSET },
#endif
#ifdef CONFIG_DEVKMEM
     [] = { "kmem", , &kmem_fops, FMODE_UNSIGNED_OFFSET },
#endif...
};

其中mem_fops对应/dev/mem节点的操作函数。

static const struct file_operations __maybe_unused mem_fops = {
    .llseek        = memory_lseek,
    .read        = read_mem,
    .write        = write_mem,-------------------------------直接对/dev/mem进行读写。
    .mmap        = mmap_mem,---------------------------------对/dev/mem进行mmap映射。
    .open        = open_mem,---------------------------------主要检查权限是否满足CAP_SYS_RAWIO。
#ifndef CONFIG_MMU
    .get_unmapped_area = get_unmapped_area_mem,
    .mmap_capabilities = memory_mmap_capabilities,
#endif
};

static int open_port(struct inode *inode, struct file *filp)
{
    return capable(CAP_SYS_RAWIO) ?  : -EPERM;
}

#define open_mem    open_port

mmap_mem()是mmap()内存映射的执行者，通过将/dev/mem对应的物理地址映射到用户空间虚拟地址。

static int mmap_mem(struct file *file, struct vm_area_struct *vma)
{
    size_t size = vma->vm_end - vma->vm_start;
    phys_addr_t offset = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT;

    /* It's illegal to wrap around the end of the physical address space. */
    if (offset + (phys_addr_t)size -  < offset)
        return -EINVAL;

    if (!valid_mmap_phys_addr_range(vma->vm_pgoff, size))return -EINVAL;

    if (!private_mapping_ok(vma))
        return -ENOSYS;

    if (!range_is_allowed(vma->vm_pgoff, size))
        return -EPERM;

    if (!phys_mem_access_prot_allowed(file, vma->vm_pgoff, size,
                        &vma->vm_page_prot))
        return -EINVAL;

    vma->vm_page_prot = phys_mem_access_prot(file, vma->vm_pgoff,
                         size,
                         vma->vm_page_prot);

    vma->vm_ops = &mmap_mem_ops;

    /* Remap-pfn-range will mark the range VM_IO */
    if (remap_pfn_range(vma,---------------------------------------------将内核中vma->vm_pgoff对应的size个页面，映射到vma区域，返回的虚拟空间起始地址是vma->vm_start。
                vma->vm_start,
                vma->vm_pgoff,
                size,
                vma->vm_page_prot)) {
        return -EAGAIN;
    }
    return ;
}

下面两个对应read()和write()两个系统调用。

static ssize_t read_mem(struct file *file, char __user *buf,
            size_t count, loff_t *ppos)
{
    phys_addr_t p = *ppos;
    ssize_t read, sz;
    void *ptr;

    if (p != *ppos)
        return ;

    if (!valid_phys_addr_range(p, count))---------------------------对输入的物理地址+大小进行验证，确保在low_memory范围内。
        return -EFAULT;
    read = ;
...
    while (count > ) {
        unsigned long remaining;
        int allowed;

        sz = size_inside_page(p, count);

        allowed = page_is_allowed(p >> PAGE_SHIFT);
        if (!allowed)
            return -EPERM;
        if (allowed == ) {
            /* Show zeros for restricted memory. */
            remaining = clear_user(buf, sz);
        } else {

            ptr = xlate_dev_mem_ptr(p);-----------------------------将物理地址转换成虚拟地址，不成功则返回-EFAULT。注意这里的地址通过_va进行转换，只有特定区域的地址才可以使用。
            if (!ptr)
                return -EFAULT;

            remaining = copy_to_user(buf, ptr, sz);-----------------将物理地址对应内容拷贝到用户空间。

            unxlate_dev_mem_ptr(p, ptr);
        }

        if (remaining)
            return -EFAULT;

        buf += sz;
        p += sz;
        count -= sz;
        read += sz;
    }

    *ppos += read;
    return read;
}

static ssize_t write_mem(struct file *file, const char __user *buf,
             size_t count, loff_t *ppos)
{
    phys_addr_t p = *ppos;
    ssize_t written, sz;
    unsigned long copied;
    void *ptr;

    if (p != *ppos)
        return -EFBIG;

    if (!valid_phys_addr_range(p, count))---------------------------确保地址在low_memory范围内。
        return -EFAULT;

    written = ;

#ifdef __ARCH_HAS_NO_PAGE_ZERO_MAPPED
    /* we don't have page 0 mapped on sparc and m68k.. */
    if (p < PAGE_SIZE) {
        sz = size_inside_page(p, count);
        /* Hmm. Do something? */
        buf += sz;
        p += sz;
        count -= sz;
        written += sz;
    }
#endif

    while (count > ) {
        int allowed;

        sz = size_inside_page(p, count);

        allowed = page_is_allowed(p >> PAGE_SHIFT);
        if (!allowed)
            return -EPERM;

        /* Skip actual writing when a page is marked as restricted. */
        if (allowed == ) {
            /*
             * On ia64 if a page has been mapped somewhere as
             * uncached, then it must also be accessed uncached
             * by the kernel or data corruption may occur.
             */
            ptr = xlate_dev_mem_ptr(p);--------------------------------__va()进行物理地址到虚拟地址的转换。
            if (!ptr) {
                if (written)
                    break;
                return -EFAULT;
            }

            copied = copy_from_user(ptr, buf, sz);
            unxlate_dev_mem_ptr(p, ptr);
            if (copied) {
                written += sz - copied;
                if (written)
                    break;
                return -EFAULT;
            }
        }

        buf += sz;
        p += sz;
        count -= sz;
        written += sz;
    }

    *ppos += written;
    return written;
}

对比mmap和read()/write()两种方式可知：

• mmap()可以读写的范围更大；read()/write()的范围局限在low_memory。
• mmap()的读写速度更快，操作更方便。

1.4 hexdump使用

devmem一次读写的内容有限，hexdump可以一次dump大量数据。

但是hexdump是通过read()/write()来获取数据，物理地址的范围受到限制。而devmem通过mmap()则没有这些限制。

hexdump -s 0x10000000 -n  /dev/mem
   fc11 18ff edf7 03fe e914 020b
 0d00 1fe4 f202  f703  e814 1cfb
 09fc 000b 06eb 07f0 ec12 01e6 11e9 03f7
 1a2d 11eb f700 ece9 eef3 05f7  eb03
 ff1a e50b 1e08 0f16 0cfa 13fb 0b06 0a1b
  fefd fd1e 0b05 f317 f9ea f00a 3ef5
 f118 fe02 f606 0f02 f1ec f4fe  eefb
 0c02 eefd f8ff 06eb 08fc f603 05fb f80e
 f6fb  f207 0a19 12ee fb0d  09f8
 fbfa  f9fe 0dfc f2fa 06fb fef4 04fa
100000a0  170e 1a05 f3f6 0c2d  0f0b 061f
100000b0  0b18 f80d ebef 05f8 f3eb  e8ff
100000c0 fb07 fdea 0efd fb02 0f10 f8f8 f016 f8f2
100000d0 130f    0b03 fc06  1e10
100000e0 011b  f7f3 fc01 f6f9 03ec 0afb ecf1
100000f0 05fb 070a f904 fbf5 f7fa  f502 0d02

2. devkmem读取内核虚拟地址空间数据

某些情况下需要读取内核某个变量的值，这时候可以通过/dev/kmem。

2.1 /dev/kmem

要使用/dev/kmem就需要在内核中打开CONFIG_DEVKMEM，menuconfig路径为：Device Drivers->Character devices->/dev/kmem virtual device support。

static const struct file_operations __maybe_unused kmem_fops = {
    .llseek        = memory_lseek,
    .read        =read_kmem,
    .write        =write_kmem,
    .mmap        =mmap_kmem,
    .open        = open_kmem,
#ifndef CONFIG_MMU
    .get_unmapped_area = get_unmapped_area_mem,
    .mmap_capabilities = memory_mmap_capabilities,
#endif
};

static int mmap_kmem(struct file *file, struct vm_area_struct *vma)
{
    unsigned long pfn;

    /* Turn a kernel-virtual address into a physical page frame */
    pfn = __pa((u64)vma->vm_pgoff << PAGE_SHIFT) >> PAGE_SHIFT;------------------将内核虚拟地址通过__pa()转换成物理地址。

    if (!pfn_valid(pfn))
        return -EIO;

    vma->vm_pgoff = pfn;
    returnmmap_mem(file, vma);
}

read_kmem()和write_kmem()需要对low_memory和high_memory进行区别对待。

对low_memory需要经过xlate_dev_kmem_ptr()后进行读写；对high_memory通过vread()/vwrite()进行读写。

static ssize_t read_kmem(struct file *file, char __user *buf,
             size_t count, loff_t *ppos)
{
    unsigned long p = *ppos;
    ssize_t low_count, read, sz;
    char *kbuf; /* k-addr because vread() takes vmlist_lock rwlock */
    int err = ;

    read = ;
    if (p < (unsigned long) high_memory) {---------------------------------------属于low_memory内存处理。
        low_count = count;
        if (count > (unsigned long)high_memory - p)
            low_count = (unsigned long)high_memory - p;--------------------------计算处于low_memory区域的内存大小。
...
        while (low_count > ) {
            sz = size_inside_page(p, low_count);

            kbuf = xlate_dev_kmem_ptr((void *)p);
            if (!virt_addr_valid(kbuf))------------------------------------------地址在PAGE_OFFSET和high_memory之间。
                return -ENXIO;

            if (copy_to_user(buf, kbuf, sz))
                return -EFAULT;
            buf += sz;
            p += sz;
            read += sz;
            low_count -= sz;
            count -= sz;
        }
    }

    if (count > ) {-------------------------------------------------------------如果还有count没处理完，那么就属于high_memory。
        kbuf = (char *)__get_free_page(GFP_KERNEL);
        if (!kbuf)
            return -ENOMEM;
        while (count > ) {
            sz = size_inside_page(p, count);-------------------------------------如果p+count不跨页，那么sz=count；否则sz只取p所在页面剩余部分大小。这样确保下面的copy_to_user()不跨页。
            if (!is_vmalloc_or_module_addr((void *)p)) {
                err = -ENXIO;
                break;
            }
            sz = vread(kbuf, (char *)p, sz);
            if (!sz)
                break;
            if (copy_to_user(buf, kbuf, sz)) {
                err = -EFAULT;
                break;
            }
            count -= sz;
            buf += sz;
            read += sz;
            p += sz;
        }
        free_page((unsigned long)kbuf);
    }
    *ppos = p;
    return read ? read : err;
}

static ssize_t do_write_kmem(unsigned long p, const char __user *buf,
                size_t count, loff_t *ppos)
{
    ssize_t written, sz;
    unsigned long copied;

    written = ;
...
    while (count > ) {
        void *ptr;

        sz = size_inside_page(p, count);

        ptr = xlate_dev_kmem_ptr((void *)p);
        if (!virt_addr_valid(ptr))
            return -ENXIO;

        copied = copy_from_user(ptr, buf, sz);
        if (copied) {
            written += sz - copied;
            if (written)
                break;
            return -EFAULT;
        }
        buf += sz;
        p += sz;
        count -= sz;
        written += sz;
    }

    *ppos += written;
    return written;
}

static ssize_t write_kmem(struct file *file, const char __user *buf,
              size_t count, loff_t *ppos)
{
    unsigned long p = *ppos;
    ssize_t wrote = ;
    ssize_t virtr = ;
    char *kbuf; /* k-addr because vwrite() takes vmlist_lock rwlock */
    int err = ;

    if (p < (unsigned long) high_memory) {
        unsigned long to_write = min_t(unsigned long, count,
                           (unsigned long)high_memory - p);
        wrote =do_write_kmem(p, buf, to_write, ppos);
        if (wrote != to_write)
            return wrote;
        p += wrote;
        buf += wrote;
        count -= wrote;
    }

    if (count > ) {
        kbuf = (char *)__get_free_page(GFP_KERNEL);
        if (!kbuf)
            return wrote ? wrote : -ENOMEM;
        while (count > ) {
            unsigned long sz = size_inside_page(p, count);
            unsigned long n;

            if (!is_vmalloc_or_module_addr((void *)p)) {
                err = -ENXIO;
                break;
            }
            n = copy_from_user(kbuf, buf, sz);
            if (n) {
                err = -EFAULT;
                break;
            }
            vwrite(kbuf, (char *)p, sz);
            count -= sz;
            buf += sz;
            virtr += sz;
            p += sz;
        }
        free_page((unsigned long)kbuf);
    }

    *ppos = p;
    return virtr + wrote ? : err;
}

2.2 devkmem

类似于devmem相对于/dev/mem，devkmem通过将/dev/kmem映射到用户空间，然后读取内容。

#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <string.h>
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/mman.h>

#define DEVKMEM         "/dev/kmem"

#define PAGE_SIZE       0x1000
#define PAGE_MASK       (~(PAGE_SIZE-1))

int main(int argc, char* argv[])
{
    int fd;
    char *mbase;
    char read_buf[];
    unsigned int varAddr, regAddr;

    varAddr = strtoul(argv[], , );

    unsigned int ptr = varAddr & ~(PAGE_MASK);

    fd = open(DEVKMEM, O_RDONLY);
    if (fd == -) {
        perror("open");
        exit(-);
    }

    mbase = mmap(,PAGE_SIZE,PROT_READ,MAP_SHARED,fd, (varAddr & PAGE_MASK));
    if (mbase == MAP_FAILED) {
        printf("map failed %s\n",strerror(errno));
    }

    printf("varAddr = 0x%X \n", varAddr);
    printf("mapbase = 0x%X \n", (unsigned int)mbase);
    printf("value   = 0x%X \n",*(unsigned int*)(mbase+ptr));

    close(fd);
    munmap(mbase,PAGE_SIZE);

    return ;
}

2.3 devkmem使用

由于devkmem需要输入地址，但由于是虚拟地址，完全地址是没有意义的。

需要通过/proc/kallsyms根据符号找到对应的内核虚拟地址，然后再通过devkmem查看其值。

比如想查看sysctl_sched_rt_runtime的值，首先查看其在内核的虚拟地址：

cat /proc/kallsyms | grep sysctl_sched_rt_runtime
808eb544 D sysctl_sched_rt_runtime

然后查看虚拟地址的值：

./devkmem 808eb544
varAddr = 0x808EB544
mapbase = 0x2ABFB000
value   = 0xE7EF0 

换算成10进制就是950000.

那么这个值对不对呢？cat /proc/sys/kernel/sched_rt_runtime_us表明结果正确。

改进点：

1.直接输入符号，显示所有符号的值。

2.不同输出格式，16进制、10进制等等。

参考文档：

《使用/dev/kmem读取内核变量的值》
《/dev/mem可没那么简单》