Linux驱动基础：msm平台，modem等framework加载

msm平台，AP和CP封装在一起，公用一块内存。所以AP需要负责把整个modem, TZ , rpm等binary拷贝到内存中以供modem等subsystem去运行。那AP这边是怎么分配这些内存，又是怎么读出来相关的binary，又如何把binary上传上去的呢？？

相关的feature

CONFIG_FW_LOADER
CONFIG_FW_LOADER_USER_HELPER
1
2

modem使用的内存申请

要设置modem的内存大小，必须首先需要确认modem binary的大小，modem需要使用的内存大小等。这个在CMA相关的内容中说过。这里在说一下高通msm8916平台，modem大小检查以及修改方法。 
1) modem binary的大小可以从以下编译的log里边看出来！！(modem_proc/build/ms目录下的pplk-XXX.log或者build_xxxx.log)。 
根据大小对齐1MB大小，就是modem binary需要流出来的大小。看如下例子里边的log，总的大小是77.04， 
所以需要在上面的dtsi文件中留出来78MB就可以。

  Image loaded at virtual address 0xc0000000
  Image:                                   55.44 MiB
  AMSS Heap:                                7.50 MiB (dynamic)
  MPSS Heap:                                4.00 MiB (dynamic)
  DSM Pools:                                5.06 MiB
  Q6Zip RO, Swap Pool:                      2.00 MiB (dynamic)
  Q6Zip RW, Swap Pool:                      1.00 MiB (dynamic)
  Q6Zip RW, dlpager Heap:                   1.00 MiB
  Extra:                                    0.54 MiB
  Pad ding:                                  0.37 MiB
  End Address Alignment:                    0.13 MiB
  Total:                                   77.04 MiB
  Available:                                7.96 MiB
1
2
3
4
5
6
7
8
9
10
11
12
13

2) 然后去修改modem_proc/config/xxx/ 目录下的cust_config.xml文件中修改modem大小

    <!-- 85 MB of physical pool-->
        <physical_pool name="DEFAULT_PHYSPOOL">
             <region base="0x88000000"  size="0x5500000" />
             <region base="0x88000000" size="0x4E00000" />
        </physical_pool>  
1
2
3
4
5

以下是modem相关的device tree的设置。这些内容也在CMA和ion内存相关的帖子里边都讲过。 
但之前有一个疑问就是，在CMA预留了一段内存之后，会把这个赋值给modem的dev->cma_area，然后在分配需要使用的内存的时候从dev->cma_area中取出来，那这个过程好像跟ion内存没有什么关系。能不能去掉下面msmxxx-ion.dtsi中 
modem_adsp_mem相关的设置呢？？ 
是可以的！！！其他几个DMA区域，如果直接从CMA分配的话，应该都可以从msmxxx-ion.dtsi文件中去掉！！ 
也就是说下面qcom,ion-heap-type = “DMA”的部分其实都可以从msm8916-ion.dtsi文件中去掉，不影响。

        //modem相关内存的device tree设置
        //pil设备相关的device tree定义
        qcom,mss@4080000 {
            compatible = "qcom,pil-q6v56-mss";
            ....
            linux,contiguous-region = <&modem_adsp_mem>;
        };

        //msmxxx-ion.dtsi定义了如下，上面说了这个部分其实是可以去掉的，不会影响相关内存的分配！！
        qcom,ion-heap@26 { /* MODEM HEAP */
            compatible = "qcom,msm-ion-reserve";
            reg = <26>;
            linux,contiguous-region = <&modem_adsp_mem>;
            qcom,ion-heap-type = "DMA";
        };

        //msmxxx-memory.dtsi定义了如下内容
        modem_adsp_mem: modem_adsp_region@0 {
            linux,reserve-contiguous-region;
            linux,reserve-region;
            linux,remove-completely;
            reg = <0x0 0x86800000 0x0 0x05800000>;
            label = "modem_adsp_mem";
        };
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

modem_adsp_mem指定的区域，需要分配出来，以供下载modem binary。

//pil_mss_driver_probe()->pil_subsys_init()
static int pil_subsys_init(struct modem_data *drv,
                    struct platform_device *pdev)
{
    ...
    drv->subsys_desc.name = "modem";
    drv->subsys_desc.dev = &pdev->dev;
    drv->subsys_desc.owner = THIS_MODULE;
    drv->subsys_desc.shutdown = modem_shutdown;
    drv->subsys_desc.powerup = modem_powerup;
    drv->subsys_desc.ramdump = modem_ramdump;
    drv->subsys_desc.free_memory = modem_free_memory;
    drv->subsys_desc.crash_shutdown = modem_crash_shutdown;
    drv->subsys_desc.err_fatal_handler = modem_err_fatal_intr_handler;
    drv->subsys_desc.stop_ack_handler = modem_stop_ack_intr_handler;
    drv->subsys_desc.wdog_bite_handler = modem_wdog_bite_intr_handler;

    drv->subsys = subsys_register(&drv->subsys_desc);

    drv->ramdump_dev = create_ramdump_device("modem", &pdev->dev);
    ...
    return ret;
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

之后在modem_powerup()的时候,会先根据modem binary的elf结构独处modem的大小等，然后计算出align之后应该的大小。 
pil_boot()-> request_firmware()读出elf头并计算大小等。 
在pil_setup_region()->pil_alloc_region()的时候，传进去的大小就是从上面读出来的大小。

pil_alloc_region min_addr = 0xc0000000 , max_addr = 0xc2b00000 , aligned_size = 0x2b00000
1

这里看着和实际的内存大小一致！！ 可能是因为留出来的CMA区域的大小正好和这个大小一致才这样的。 
在实际调试过程中，也可以打印这个大小之后，调整CMA大小。

再看看实际的CMA大小是怎么申请的。

//调用顺序
pil_boot()->pil_init_mmap()->pil_setup_region()->pil_alloc_region()->
dma_alloc_attrs()->arm_dma_alloc()->__dma_alloc()->__alloc_from_contiguous()->
1
2
3

这个调用的顺序，一步一步往下看可以看到，实际上分配的区域是一块CMA区域，而且就是在CMA注册之后，在相应的platform设备注册的时候保存到dev->cma_area中的区域。 
在相应的设备注册的时候，如果设备的device tree中有”linux,contiguous-region”的时候，就会寻找相应的CMA区域并进行保留。这都是因为注册了platform_bus_typ的notifier函数

    bus_register_notifier(&platform_bus_type, &cma_dev_init_nb);
1

看下面的log。

<6>[0.487642]  [0:swapper/0:1] cma: Assigned CMA region at 0 to 1de0000.qcom,venus device
<6>[0.489469]  [0:swapper/0:1] cma: Assigned CMA region at 0 to 4080000.qcom,mss device
<6>[0.490756]  [0:swapper/0:1] cma: Assigned CMA region at 0 to a21b000.qcom,pronto device
<6>[1.125342]  [0:swapper/0:1] cma: Assigned CMA region at 0 to 8.qcom,ion-heap device
<6>[1.125793]  [0:swapper/0:1] cma: Assigned CMA region at 0 to 1b.qcom,ion-heap device
<6>[1.126233]  [0:swapper/0:1] cma: Assigned CMA region at 0 to 1c.qcom,ion-heap device
<6>[1.126671]  [0:swapper/0:1] cma: Assigned CMA region at 0 to 17.qcom,ion-heap device
<6>[1.127298]  [0:swapper/0:1] cma: Assigned CMA region at 0 to 1a.qcom,ion-heap device
1
2
3
4
5
6
7
8

这里看到4080000.qcom,mss这个device相应的区域已经保留了CMA区域。 
然后在上面进行分配的时候，在 
__alloc_from_contiguous()->dma_alloc_from_contiguous()->dev_get_cma_area()函数中取到 
相应的dev->cma_area。

modem相关内存的使用和下载

pil_load_seg()->request_firmware_direct()->_request_firmware()函数身生成相应的device节点，并通知ueventd去读取相应的binary然后下载。以下是pil_load_seg里边打印的正在试图下载的binary。

<6>[29.129737]  [1:init:1] pil_load_seg fw_name = modem.b02
<6>[29.157808]  [1:init:1] pil_load_seg fw_name = modem.b07
<6>[29.191477]  [1:init:1] pil_load_seg fw_name = modem.b17
<6>[29.348480]  [1:init:1] pil_load_seg fw_name = modem.b19
<6>[29.409733]  [1:init:1] pil_load_seg fw_name = modem.b20
<6>[29.489639]  [1:init:1] pil_load_seg fw_name = modem.b23
<6>[29.519624]  [1:init:1] pil_load_seg fw_name = modem.b24
<6>[29.549829]  [1:init:1] pil_load_seg fw_name = modem.b25
<6>[29.591918]  [1:init:1] pil_load_seg fw_name = modem.b27
<6>[31.997036]  [0:wcnss_service:307] pil_load_seg fw_name = wcnss.b02
<6>[32.658390]  [0:wcnss_service:307] pil_load_seg fw_name = wcnss.b04
<6>[32.693754]  [0:wcnss_service:307] pil_load_seg fw_name = wcnss.b06
<6>[32.848104]  [3:wcnss_service:307] pil_load_seg fw_name = wcnss.b09
<6>[32.854061]  [3:wcnss_service:307] pil_load_seg fw_name = wcnss.b10
<6>[32.876115]  [3:wcnss_service:307] pil_load_seg fw_name = wcnss.b11
<6>[37.384287]  [1:TimedEventQueue:771] pil_load_seg fw_name = venus.b02
<6>[37.438222]  [1:TimedEventQueue:771] pil_load_seg fw_name = venus.b03
<6>[37.484909]  [2:TimedEventQueue:771] pil_load_seg fw_name = venus.b04
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

在 _request_firmware()->fw_load_from_user_helper()->_request_firmware_load()函数中就在生成相应的dev节点，并通知ueventd。

static int _request_firmware_load(struct firmware_priv *fw_priv, bool uevent,
                  long timeout)
{
    int retval = 0;
    struct device *f_dev = &fw_priv->dev;
    struct firmware_buf *buf = fw_priv->buf;
    struct bin_attribute *fw_attr_data = buf->dest_addr ?
            &firmware_direct_attr_data : &firmware_attr_data;

    /* fall back on userspace loading */
    buf->is_paged_buf = buf->dest_addr ? false : true;

    dev_set_uevent_suppress(f_dev, true);

    /* Need to pin this module until class device is destroyed */
    __module_get(THIS_MODULE);

    retval = device_add(f_dev);

    //以下生成的data和loading节点，用于ueventd读取相应的binary，然后通过节点加载到内存的。

    //用于下载的节点，
    retval = device_create_bin_file(f_dev, fw_attr_data);

    //生成一个loading的节点，loading节点用于控制的
    retval = device_create_file(f_dev, &dev_attr_loading);
    if (retval) {
        dev_err(f_dev, "%s: device_create_file failed\n", __func__);
        goto err_del_bin_attr;
    }

    if (uevent) { //这里正在通知ueventd
        dev_set_uevent_suppress(f_dev, false);
        dev_dbg(f_dev, "firmware: requesting %s\n", buf->fw_id);
        if (timeout != MAX_SCHEDULE_TIMEOUT)
            schedule_delayed_work(&fw_priv->timeout_work, timeout);
        kobject_uevent(&fw_priv->dev.kobj, KOBJ_ADD);
    }
    wait_for_completion(&buf->completion);
    cancel_delayed_work_sync(&fw_priv->timeout_work);
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

_request_firmware() -> assign_firmware_buf() 这是做什么的？？

来看一下ueventd.c文件中是怎么检测这个然后读binary，通过loading节点加载binary的。

int ueventd_main(int argc, char **argv){

    ...
    while(1) {
        ufd.revents = 0;
        nr = poll(&ufd, 1, -1);
        if (nr <= 0)
            continue;
        if (ufd.revents & POLLIN)
               handle_device_fd();
    }
}

void handle_device_fd(){
    ...
    handle_firmware_event(&uevent);//process_firmware_event()
}

#define SYSFS_PREFIX    "/sys"
#define FIRMWARE_DIR1   "/etc/firmware"
#define FIRMWARE_DIR2   "/vendor/firmware"
#define FIRMWARE_DIR3   "/firmware/image"
#define FIRMWARE_DIR4   "/firmware-modem/image"
#define DEVICES_BASE    "/devices/soc.0"

static void process_firmware_event(struct uevent *uevent){
    ...
    l = asprintf(&root, SYSFS_PREFIX"%s/", uevent->path);
    l = asprintf(&loading, "%sloading", root);
    l = asprintf(&file1, FIRMWARE_DIR1"/%s", uevent->firmware);
    l = asprintf(&file2, FIRMWARE_DIR2"/%s", uevent->firmware);
    l = asprintf(&file3, FIRMWARE_DIR3"/%s", uevent->firmware);
    l = asprintf(&file4, FIRMWARE_DIR4"/%s", uevent->firmware);

    loading_fd = open(loading, O_WRONLY);
    ...
    if(!load_firmware(fw_fd, loading_fd, data_fd)) //加载binary
        INFO("firmware: copy success { '%s', '%s' }\n", root, uevent->firmware);
    else
        INFO("firmware: copy failure { '%s', '%s' }\n", root, uevent->firmware);

}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

以下看看ueventd中，真正把读到的binary，传给kernel的函数

static int load_firmware(int fw_fd, int loading_fd, int data_fd)
{
    struct stat st;
    long len_to_copy;
    int ret = 0;

    //fstat查看binary的信息，读出来size等
    if(fstat(fw_fd, &st) < 0)
        return -1;
    len_to_copy = st.st_size;

    write(loading_fd, "1", 1);  /* start transfer */

    while (len_to_copy > 0) {
        char buf[PAGE_SIZE];
        ssize_t nr;
        //读
        nr = read(fw_fd, buf, sizeof(buf));
        if(!nr)
            break;
        if(nr < 0) {
            ret = -1;
            break;
        }

        len_to_copy -= nr;
        while (nr > 0) {
            ssize_t nw = 0;
            //写到data节点
            nw = write(data_fd, buf + nw, nr);
            if(nw <= 0) {
                ret = -1;
                goto out;
            }
            nr -= nw;
        }
    }

out:
    if(!ret) //loading节点用于通知kernel加载情况！！
        write(loading_fd, "0", 1);  /* successful end of transfer */
    else
        write(loading_fd, "-1", 2); /* abort transfer */

    return ret;
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

内核中，data节点出来write的函数在_request_firmware_load()中根据buf->dest_addr的值有所不同

static int _request_firmware_load(struct firmware_priv *fw_priv, bool uevent,
                  long timeout)
{
    ...
    struct bin_attribute *fw_attr_data = buf->dest_addr ?
            &firmware_direct_attr_data : &firmware_attr_data;
    ...

}
1
2
3
4
5
6
7
8
9

在下载modem.bxx的时候应该都是有buf->dest_addr才对

<6>[29.355860]  [0:init:1] pil_load_seg fw_name = modem.b02
<6>[29.361829]  [0:init:1] fw_load_from_user_helper start
<6>[29.368051]  [0:init:1] _request_firmware_load buf->dest_addr = 0x86800000
<6>[29.380308]  [0:ueventd:230] firmware_loading_store started
...
<6>[29.391942]  [0:init:1] pil_load_seg fw_name = modem.b07
<6>[29.398801]  [0:init:1] fw_load_from_user_helper start
<6>[29.404996]  [0:init:    1] _request_firmware_load buf->dest_addr = 0x86840000
...
1
2
3
4
5
6
7
8
9

//write(data_fd, buf + nw, nr); buf对应buffer？ offset? count对应nr??
static ssize_t firmware_direct_write(struct file *filp, struct kobject *kobj,
                   struct bin_attribute *bin_attr,
                   char *buffer, loff_t offset, size_t count)
{
    struct device *dev = kobj_to_dev(kobj);
    struct firmware_priv *fw_priv = to_firmware_priv(dev);
    //获取uevent读取modem binary时候读到的内容，保存在firmware_priv中。firmware_priv中的firmware_buf保存了binary的物理地址，大小等等信息

    struct firmware *fw;
    ssize_t retval;

    if (!capable(CAP_SYS_RAWIO))
        return -EPERM;

    mutex_lock(&fw_lock);
    fw = fw_priv->fw;
    if (!fw || test_bit(FW_STATUS_DONE, &fw_priv->buf->status)) {
        retval = -ENODEV;
        goto out;
    }

    retval = __firmware_data_rw(fw_priv, buffer, &offset, count, 0);
    if (retval < 0)
        goto out;

    fw_priv->buf->size = max_t(size_t, offset, fw_priv->buf->size);
out:
    mutex_unlock(&fw_lock);
    return retval;
}

static int __firmware_data_rw(struct firmware_priv *fw_priv, char *buffer,
                loff_t *offset, size_t count, int read)
{
    u8 __iomem *fw_buf; 

    struct firmware_buf *buf = fw_priv->buf;
    int retval = count;

    if ((*offset + count) > buf->dest_size) {
        pr_debug("%s: Failed size check.\n", __func__);
        retval = -EINVAL;
        goto out;
    }

    //fw_buf 就是要拷贝到内存中的modem binary物理地址对应的虚拟地址。
    //map_fw_mem函数中，会根据虚拟地址以及需要拷贝的大小，map出一段虚拟地址。
    //map一段物理地址，然后返回内核可以访问的虚拟地址，，这个是通过ioremap相关的函数实现的

    fw_buf = buf->map_fw_mem(buf->dest_addr + *offset, count,
                    buf->map_data);
    if (!fw_buf) {
        pr_debug("%s: Failed ioremap.\n", __func__);
        retval = -ENOMEM;
        goto out;
    }

    //读写，直接拷贝就可以
    if (read)
        memcpy(buffer, fw_buf, count);
    else
        memcpy(fw_buf, buffer, count);

    *offset += count;
    buf->unmap_fw_mem(fw_buf, count, buf->map_data);

out:
    return retval;
}

static void *map_fw_mem(phys_addr_t paddr, size_t size, void *data)
{
    struct pil_map_fw_info *info = data;

    return dma_remap(info->dev, info->region, paddr, size,
                    &info->attrs);
}

static inline void *dma_remap(struct device *dev, void *cpu_addr,
        dma_addr_t dma_handle, size_t size, struct dma_attrs *attrs)
{
    const struct dma_map_ops *ops = get_dma_ops(dev);
    BUG_ON(!ops);

    if (!ops->remap) {
        WARN_ONCE(1, "Remap function not implemented for %pS\n",
                ops->remap);
        return NULL;
    }

    return ops->remap(dev, cpu_addr, dma_handle, size, attrs);
}

static void *arm_dma_remap(struct device *dev, void *cpu_addr,
            dma_addr_t handle, size_t size,
            struct dma_attrs *attrs)
{
    struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
    pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
    unsigned long offset = handle & ~PAGE_MASK;

    size = PAGE_ALIGN(size + offset);
    return __dma_alloc_remap(page, size, GFP_KERNEL, prot,
                    __builtin_return_address(0)) + offset;

}

static void *
__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
    const void *caller)
{
    struct vm_struct *area;
    unsigned long addr;

    /*
     * DMA allocation can be mapped to user space, so lets
     * set VM_USERMAP flags too.
     */
    //得到一段满足要求的vm_struct。这里
    area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP,
                  caller);
    if (!area)
        return NULL;
    addr = (unsigned long)area->addr;
    area->phys_addr = __pfn_to_phys(page_to_pfn(page));

    //addr是得到的vm_struct对应的虚拟地址，内核可以访问的
    //所以根据物理地址以及对应的虚拟地址以及大小等情况，ioremap_page_range会做一个page table
    //这样内核就可以直接访问这段内存
    if (ioremap_page_range(addr, addr + size, area->phys_addr, prot)) {
        vunmap((void *)addr);
        return NULL;
    }
    return (void *)addr;//返回虚拟内存，现在这个虚拟内存就可以直接访问了
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136

和ioremap_page_range()比较像。