IO拆分流程分析

概述

对于硬盘的访问，如果IO所涉及的数据量太大、或者跨页等原因，有可能需要对访问IO进行拆分成多个小IO来访问。本文根据SPDK开源代码中example\nvme\hello_world示例进行研究

Request结构

在SPDK中将对硬盘的访问IO首先包装成一个Request请求，如果此IO需要拆分，则会将拆分后的IO记录到未拆分时创建的这个Request的children字段（队列），下面是Request结构，只显示几个重要的以及跟拆分IO相关的字段

struct nvme_request {
    struct spdk_nvme_cmd        cmd; // SQE
    ......
    /**
     * Number of children requests still outstanding for this
     *  request which was split into multiple child requests.
     */
    uint16_t            num_children;
    ......
    struct spdk_nvme_qpair      *qpair; // IO Qpair
    ......
    struct spdk_nvme_cpl        cpl; // CQE
    /**
     * The following members should not be reordered with members
     *  above.  These members are only needed when splitting
     *  requests which is done rarely, and the driver is careful
     *  to not touch the following fields until a split operation is
     *  needed, to avoid touching an extra cacheline.
     */

    /**
     * Points to the outstanding child requests for a parent request.
     *  Only valid if a request was split into multiple children
     *  requests, and is not initialized for non-split requests.
     */
    TAILQ_HEAD(, nvme_request)  children;

    /**
     * Linked-list pointers for a child request in its parent's list.
     */
    TAILQ_ENTRY(nvme_request)   child_tailq;

    /**
     * Points to a parent request if part of a split request,
     *   NULL otherwise.
     */
    struct nvme_request     *parent;

    /**
     * Completion status for a parent request.  Initialized to all 0's
     *  (SUCCESS) before child requests are submitted.  If a child
     *  request completes with error, the error status is copied here,
     *  to ensure that the parent request is also completed with error
     *  status once all child requests are completed.
     */
    struct spdk_nvme_cpl        parent_status;

    /**
     * The user_cb_fn and user_cb_arg fields are used for holding the original
     * callback data when using nvme_allocate_request_user_copy.
     */
    spdk_nvme_cmd_cb        user_cb_fn;
    void                *user_cb_arg;
    void                *user_buffer;
};

IO拆分

接口调用关系

下图是一个IO的request的创建以及拆分动作的调用过程。

实现

拆分过程最主要的就是一个while循环，将一个大的IO拆分成多个能一次处理的小IO

吐槽：接口的参数不是一般的多。。。

static struct nvme_request *
_nvme_ns_cmd_split_request(struct spdk_nvme_ns *ns,
               struct spdk_nvme_qpair *qpair,
               const struct nvme_payload *payload,
               uint32_t payload_offset, uint32_t md_offset,
               uint64_t lba, uint32_t lba_count,
               spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t opc,
               uint32_t io_flags, struct nvme_request *req,
               uint32_t sectors_per_max_io, uint32_t sector_mask,
               uint16_t apptag_mask, uint16_t apptag, int *rc)
{
    uint32_t        sector_size = _nvme_get_host_buffer_sector_size(ns, io_flags);
    uint32_t        remaining_lba_count = lba_count;
    struct nvme_request *child;

    while (remaining_lba_count > 0) {
        lba_count = sectors_per_max_io - (lba & sector_mask);
        lba_count = spdk_min(remaining_lba_count, lba_count);

        child = _nvme_add_child_request(ns, qpair, payload, payload_offset, md_offset,
                        lba, lba_count, cb_fn, cb_arg, opc,
                        io_flags, apptag_mask, apptag, req, true, rc);
        if (child == NULL) {
            return NULL;
        }

        remaining_lba_count -= lba_count;
        lba += lba_count;
        payload_offset += lba_count * sector_size;
        md_offset += lba_count * ns->md_size;
    }

    return req;
}

而对于每一个小的IO，都会调用接口_nvme_ns_cmd_rw()创建成一个request，然后调用nvme_request_add_child()将新的request放到到最开始的大IO对应的Request的一个子request队列中，如下

static inline void
nvme_request_add_child(struct nvme_request *parent, struct nvme_request *child)
{
    ......
    
    parent->num_children++;
    TAILQ_INSERT_TAIL(&parent->children, child, child_tailq); // 链接到parent的children队列
    child->parent = parent;
    child->cb_fn = nvme_cb_complete_child;
    child->cb_arg = child;
}

从上面代码中可以看出，每一个子request完成后的回调接口都是指向nvme_cb_complete_child，这个接口做了一些资源清理的工作，如下：

static inline void
nvme_cb_complete_child(void *child_arg, const struct spdk_nvme_cpl *cpl)
{
    struct nvme_request *child = child_arg;
    struct nvme_request *parent = child->parent;

    nvme_request_remove_child(parent, child);

    if (spdk_nvme_cpl_is_error(cpl)) {
        memcpy(&parent->parent_status, cpl, sizeof(*cpl));
    }

    if (parent->num_children == 0) {
        nvme_complete_request(parent->cb_fn, parent->cb_arg, parent->qpair,
                      parent, &parent->parent_status);
        nvme_free_request(parent);
    }
}

IO执行

前面将IO拆分后组成多个小的request放到父request的一个队列中。对父request进行submit，在处理时会判断父request中是否存在子request，如果有则会循环将子request进行submit，所有子request完成之后直接退出不会再处理父request（相当于父request只是一个容器），代码如下：

static inline int
_nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
{
    ......
    
    if (req->num_children) {
        /*
         * This is a split (parent) request. Submit all of the children but not the parent
         * request itself, since the parent is the original unsplit request.
         */
        TAILQ_FOREACH_SAFE(child_req, &req->children, child_tailq, tmp) {
            if (spdk_likely(!child_req_failed)) {
                rc = nvme_qpair_submit_request(qpair, child_req);
                if (spdk_unlikely(rc != 0)) {
                    child_req_failed = true;
                }
            } else { /* free remaining child_reqs since one child_req fails */
                nvme_request_remove_child(req, child_req);
                nvme_request_free_children(child_req);
                nvme_free_request(child_req);
            }
        }

        if (spdk_unlikely(child_req_failed)) {
            /* part of children requests have been submitted,
             * return success since we must wait for those children to complete,
             * but set the parent request to failure.
             */
            if (req->num_children) {
                req->cpl.status.sct = SPDK_NVME_SCT_GENERIC;
                req->cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
                return 0;
            }
            goto error;
        }

        return rc;
    }

    ......
}

IO拆分分析

SPDK中IO拆分条件

SPDK中调用IO拆分的点如下代码：

static inline struct nvme_request *
_nvme_ns_cmd_rw(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
        const struct nvme_payload *payload, uint32_t payload_offset, uint32_t md_offset,
        uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t opc,
        uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag, bool check_sgl, int *rc)
{
    ......

    /*
     * Intel DC P3*00 NVMe controllers benefit from driver-assisted striping.
     * If this controller defines a stripe boundary and this I/O spans a stripe
     *  boundary, split the request into multiple requests and submit each
     *  separately to hardware.
     */
    if (sectors_per_stripe > 0 &&
        (((lba & (sectors_per_stripe - 1)) + lba_count) > sectors_per_stripe)) {

        return _nvme_ns_cmd_split_request(ns, qpair, payload, payload_offset, md_offset, lba, lba_count,
                          cb_fn,
                          cb_arg, opc,
                          io_flags, req, sectors_per_stripe, sectors_per_stripe - 1, apptag_mask, apptag, rc);
    } else if (lba_count > sectors_per_max_io) {
        return _nvme_ns_cmd_split_request(ns, qpair, payload, payload_offset, md_offset, lba, lba_count,
                          cb_fn,
                          cb_arg, opc,
                          io_flags, req, sectors_per_max_io, 0, apptag_mask, apptag, rc);
    } else if (nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_SGL && check_sgl) {
        if (ns->ctrlr->flags & SPDK_NVME_CTRLR_SGL_SUPPORTED) {
            return _nvme_ns_cmd_split_request_sgl(ns, qpair, payload, payload_offset, md_offset,
                                  lba, lba_count, cb_fn, cb_arg, opc, io_flags,
                                  req, apptag_mask, apptag, rc);
        } else {
            return _nvme_ns_cmd_split_request_prp(ns, qpair, payload, payload_offset, md_offset,
                                  lba, lba_count, cb_fn, cb_arg, opc, io_flags,
                                  req, apptag_mask, apptag, rc);
        }
    }

    _nvme_ns_cmd_setup_request(ns, req, opc, lba, lba_count, io_flags, apptag_mask, apptag);
    return req;
}

代码中只有三个拆分IO的分支：

磁盘有设置stripe，并且此次IO跨stripe边界了
IO中LBA数量超过磁盘的一个IO支持最大的sector数量
这个很奇怪，看最外层条件是设置使用SGL方式，但是内部条件又分成了SGL和RPR两种方式。。。

磁盘支持的最大IO计算方法

在SPDK中有两个结构体中的几个字段记录了磁盘支持的IO相关的信息(在controller初始化过程中会获取相关信息)如下

struct spdk_nvme_ctrlr {
    ......
    /** maximum i/o size in bytes */
    uint32_t            max_xfer_size;

    /** minimum page size supported by this controller in bytes */
    uint32_t            min_page_size;

    /** selected memory page size for this controller in bytes */
    uint32_t            page_size;
    ......
};

struct spdk_nvme_ns {
    struct spdk_nvme_ctrlr      *ctrlr;
    uint32_t            sector_size;

    /*
     * Size of data transferred as part of each block,
     * including metadata if FLBAS indicates the metadata is transferred
     * as part of the data buffer at the end of each LBA.
     */
    uint32_t            extended_lba_size;

    ......
    uint32_t            sectors_per_max_io;
    uint32_t            sectors_per_max_io_no_md;
    uint32_t            sectors_per_stripe;
    ......
};

其中page_size是通过读取controller的CAP信息获取，如下

static void
nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr *ctrlr)
{
    ......
    ctrlr->min_page_size = 1u << (12 + ctrlr->cap.bits.mpsmin);

    /* For now, always select page_size == min_page_size. */
    ctrlr->page_size = ctrlr->min_page_size;
    ......
}

字段max_xfer_size字段最大值依赖NVMe支持的prp_entry_size和page_size计算方式如下：

#define NVME_MAX_PRP_LIST_ENTRIES   (503)

static  uint32_t
nvme_pcie_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr)
{
    /*
     * For commands requiring more than 2 PRP entries, one PRP will be
     *  embedded in the command (prp1), and the rest of the PRP entries
     *  will be in a list pointed to by the command (prp2).  The number
     *  of PRP entries in the list is defined by
     *  NVME_MAX_PRP_LIST_ENTRIES.
     *
     *  Note that the max xfer size is not (MAX_ENTRIES + 1) * page_size
     *  because the first PRP entry may not be aligned on a 4KiB
     *  boundary.
     */
    return NVME_MAX_PRP_LIST_ENTRIES * ctrlr->page_size;
}

而max_xfer_size在系统中使用的真实值，还依赖于controller支持的mdts，nvme协议中controller的mdts(单位应该是page个数)定义如下：

MDTS

max_xfer_size最终值的计算如下：

static void
nvme_ctrlr_identify_done(void *arg, const struct spdk_nvme_cpl *cpl)
{
    ......
    /*
     * Use MDTS to ensure our default max_xfer_size doesn't exceed what the
     *  controller supports.
     */
    ctrlr->max_xfer_size = nvme_transport_ctrlr_get_max_xfer_size(ctrlr);
    NVME_CTRLR_DEBUGLOG(ctrlr, "transport max_xfer_size %u\n", ctrlr->max_xfer_size);
    if (ctrlr->cdata.mdts > 0) {
        ctrlr->max_xfer_size = spdk_min(ctrlr->max_xfer_size,
                        ctrlr->min_page_size * (1 << ctrlr->cdata.mdts));
        NVME_CTRLR_DEBUGLOG(ctrlr, "MDTS max_xfer_size %u\n", ctrlr->max_xfer_size);
    }
    ......
}

spdk_nvme_ns->sector_size字段是由namespace的LBAF->LBADS信息获取，代码如下

void
nvme_ns_set_identify_data(struct spdk_nvme_ns *ns)
{
    struct spdk_nvme_ns_data    *nsdata;

    nsdata = _nvme_ns_get_data(ns);

    ns->flags = 0x0000;

    ns->sector_size = 1 << nsdata->lbaf[nsdata->flbas.format].lbads;
    ns->extended_lba_size = ns->sector_size;
    ......
}

namespace的LBAF->LBADS在nvme协议中的定义如下：

LBADS

从上面信息看，sector_size表示一个block的大小以字节为单位，最小是512字节；max_xfer_size表示一个IO最大的字节数；从这两个数据就可以知道一个IO最多可以有多少个block，计算如下：

void
nvme_ns_set_identify_data(struct spdk_nvme_ns *ns)
{
    ......

    ns->sectors_per_max_io = spdk_nvme_ns_get_max_io_xfer_size(ns) / ns->extended_lba_size;
    ns->sectors_per_max_io_no_md = spdk_nvme_ns_get_max_io_xfer_size(ns) / ns->sector_size;
    if (ns->ctrlr->quirks & NVME_QUIRK_MDTS_EXCLUDE_MD) {
        ns->sectors_per_max_io = ns->sectors_per_max_io_no_md;
    }
    ......
}

对于字段ns->sectors_per_stripe表示controller/namespace自定义的一个带边界的信息，如果访问的IO跨边界也是需要进行拆分（如前面讲SPDK拆分中的第一个条件，当前已知的是Intel的DC P3*00 NVMe controllers有此特性）

此字段的配置方式如下：

void
nvme_ns_set_identify_data(struct spdk_nvme_ns *ns)
{
    ......
    if (nsdata->noiob) {
        ns->sectors_per_stripe = nsdata->noiob;
        SPDK_DEBUGLOG(nvme, "ns %u optimal IO boundary %" PRIu32 " blocks\n",
                  ns->id, ns->sectors_per_stripe);
    } else if (ns->ctrlr->quirks & NVME_INTEL_QUIRK_STRIPING &&
           ns->ctrlr->cdata.vs[3] != 0) {
        ns->sectors_per_stripe = (1ULL << ns->ctrlr->cdata.vs[3]) * ns->ctrlr->min_page_size /
                     ns->sector_size;
        SPDK_DEBUGLOG(nvme, "ns %u stripe size quirk %" PRIu32 " blocks\n",
                  ns->id, ns->sectors_per_stripe);
    } else {
        ns->sectors_per_stripe = 0;
    }
    ......
}

从上面代码可知，第一个条件是namespace中的一个字段noiob，nvme协议中定义如下:

NOIOB

第二个条件是controller的一个字段vs（vendor specific，这里使用vs[3]应该就是intel DC P3*00 NVMe controllers的定义），nvme协议中定义如下:

vendor specific

构造验证IO拆分

SPDK的hello_world示例中，可以针对第二个拆分条件进行构造，hello_world.c文件做如下修改

申请buffer时申请2M的空间，如下

sequence.buf = spdk_zmalloc(0x200000, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);

写IO时入参lba_count改为4096

rc = spdk_nvme_ns_cmd_write(ns_entry->ns, ns_entry->qpair, sequence.buf,
                      0, /* LBA start */
                      4096, /* number of LBAs */
                      write_complete, &sequence, 0);

经过调试，如上构造满足SPDK的IO拆分条二个条件lba_count > sectors_per_max_io

在前面IO执行章节的_nvme_qpair_submit_request接口中增加打印children的数量，如下

SPDK_ERRLOG("----------num of children = %d------------\n\n", req->num_children);

         if (req->num_children) {
            ......
         }

打印子request的个数如下

[2023-03-23 09:02:41.777381] nvme_qpair.c: 946:_nvme_qpair_submit_request: *ERROR*:----------num of children = 2------------

前面lba_count改为4096（其实大于2048就可以）可以进行拆分的原因如下，经过调试，在controller初始化完之后对应支持IO大小的相关信息如下：

namespace相关

(gdb) p *ns
$2 = {
    ctrlr = 0x2000003d60c0, 
    sector_size = 512,  // 一个block占用的字节数:1 << nsdata->lbaf[0].lbads
    extended_lba_size = 512, 
    md_size = 0, 
    pi_type = 0, 
    sectors_per_max_io = 2048,  // 即一个IO最多有2048个block，所以lba_count大于这个数之后会拆分
    sectors_per_max_io_no_md = 2048, 
    ......
    nsdata = {
        ...... 
        lbaf = {{ms = 0, lbads = 9, rp = 0, reserved6 = 0}, {ms = 0, lbads = 0, rp = 0, reserved6 = 0} <repeats 15 times>}, 
        reserved6 = '\000' <repeats 191 times>,
        vendor_specific = '\000' <repeats 3711 times>
    }
    ......
    node = {rbe_left = 0x0, rbe_right = 0x2000002f2e00, rbe_parent = 0x1}
}

controller相关

ctrlr (after identify cmd)
{
    ......
    max_xfer_size = 1048576, // MIN(page_size * 503, page_size * (1<<cdata->mdts))
    min_page_size = 4096, 
    page_size = 4096, 
    ...... 
    cdata = {
        vid = 5549, 
        ssvid = 5549,
        sn = "VMware NVME_0000\000\000\000", 
        mn = "VMware Virtual NVMe Disk", '\000' <repeats 15 times>, 
        fr = "1.3\000\000\000\000", 
        rab = 0 '\000',
        ieee = "\000PV", 
        cmic = {multi_port = 0 '\000', multi_ctrlr = 0 '\000', sr_iov = 0 '\000', ana_reporting = 0 '\000', reserved = 0 '\000'},
        mdts = 8 '\b',
        ......
        vs = '\000' <repeats 1023 times>
    },
    ......
}

最后编辑于：2023.04.04 14:18:13

©著作权归作者所有,转载或内容合作请联系作者
【社区内容提示】社区部分内容疑似由AI辅助生成，浏览时请结合常识与多方信息审慎甄别。
平台声明：文章内容（如有图片或视频亦包括在内）由作者上传并发布，文章内容仅代表作者本人观点，简书系信息发布平台，仅提供信息存储服务。

IO拆分流程分析

IO拆分流程分析

概述

Request结构

IO拆分

接口调用关系

实现

IO执行

IO拆分分析

SPDK中IO拆分条件

磁盘支持的最大IO计算方法

构造验证IO拆分

相关阅读更多精彩内容

友情链接更多精彩内容