最近在代码中使用了Linux AIO接口，其需要通过O_DIRECT方式打开文件，同时在IO时要求块大小对齐。
对于块大小对齐，找到了一个解释：Why does O_DIRECT require I/O to be 512-byte aligned?
O_DIRECT应该是要求buffer地址，传送数据大小，文件offset都是512-byte对齐，即ioctl BLKSSZGET 获取的大小，和mkfs时的block大小无关。
实际测试的结果也符合这个解释：
[root@localhost ~]# mkfs.ext4 -b 4096 /dev/sdd2
[root@localhost ~]# umpe2fs /dev/sdd2 | grep -i 'block size'
dumpe2fs 1.42.9 (28-Dec-2013)
Block size: 4096
[root@localhost 4096] ./readwrite.out
1. test count.
write 4096 bytes, ret:4096.
write 2048 bytes, ret:2048.
write 1024 bytes, ret:1024.
write 512 bytes, ret:512.
write 256 bytes failed. ret:-1, Invalid argument
2. test offset.
write offset 4096, ret:512.
write offset 2048, ret:512.
write offset 1024, ret:512.
write offset 512, ret:512.
write offset 256 failed. ret:-1, Invalid argument
看内核代码是在do_blockdev_direct_IO函数中判断对齐并返回-EINVAL
static inline ssize_t
do_blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
struct block_device *bdev, const struct iovec *iov, loff_t offset,
unsigned long nr_segs, get_block_t get_block, dio_iodone_t end_io,
dio_submit_t submit_io, int flags)
{
ssize_t retval = -EINVAL;
/* 省略掉无关代码*/
if (rw & WRITE)
rw = WRITE_ODIRECT;
/*
* Avoid references to bdev if not absolutely needed to give
* the early prefetch in the caller enough time.
*/
if (offset & blocksize_mask) {
if (bdev)
blkbits = blksize_bits(bdev_logical_block_size(bdev));
blocksize_mask = (1 << blkbits) - 1;
if (offset & blocksize_mask)
goto out;
}
/* Check the memory alignment. Blocks cannot straddle pages */
for (seg = 0; seg < nr_segs; seg++) {
addr = (unsigned long)iov[seg].iov_base;
size = iov[seg].iov_len;
end += size;
if (unlikely((addr & blocksize_mask) ||
(size & blocksize_mask))) {
if (bdev)
blkbits = blksize_bits(
bdev_logical_block_size(bdev));
blocksize_mask = (1 << blkbits) - 1;
if ((addr & blocksize_mask) || (size & blocksize_mask))
goto out;
}
}
通过kprobe可以确认是在这个函数中
cd /sys/kernel/debug/tracing
echo 'r:myretprobe do_blockdev_direct_IO $retval' > kprobe_events
echo 1 > events/kprobes/myretprobe/enable
➜ tracing more trace
# tracer: nop
#
# entries-in-buffer/entries-written: 9/9 #P:2
#
# _-----=> irqs-off
# / _----=> need-resched
# | / _---=> hardirq/softirq
# || / _--=> preempt-depth
# ||| / delay
# TASK-PID CPU# |||| TIMESTAMP FUNCTION
# | | | |||| | |
<...>-67363 [000] d..1 94189.878965: myretprobe: (__blockdev_direct_IO+0x55/0x60 <- do_blockdev_direct_IO) arg1=1000
<...>-67363 [000] d..1 94189.880246: myretprobe: (__blockdev_direct_IO+0x55/0x60 <- do_blockdev_direct_IO) arg1=800
<...>-67363 [000] d..1 94189.881208: myretprobe: (__blockdev_direct_IO+0x55/0x60 <- do_blockdev_direct_IO) arg1=400
<...>-67363 [000] d..1 94189.882268: myretprobe: (__blockdev_direct_IO+0x55/0x60 <- do_blockdev_direct_IO) arg1=200
<...>-67363 [000] d..1 94189.883792: myretprobe: (__blockdev_direct_IO+0x55/0x60 <- do_blockdev_direct_IO) arg1=200
<...>-67363 [000] d..1 94189.884805: myretprobe: (__blockdev_direct_IO+0x55/0x60 <- do_blockdev_direct_IO) arg1=200
<...>-67363 [000] d..1 94189.885609: myretprobe: (__blockdev_direct_IO+0x55/0x60 <- do_blockdev_direct_IO) arg1=200
<...>-67363 [000] d..1 94189.887123: myretprobe: (__blockdev_direct_IO+0x55/0x60 <- do_blockdev_direct_IO) arg1=200
<...>-67363 [000] d..1 94189.887222: myretprobe: (__blockdev_direct_IO+0x55/0x60 <- do_blockdev_direct_IO) arg1=ffffffffffffffea
完整测试代码：
#define _GNU_SOURCE
#include <unistd.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
#include <fcntl.h>
#include <string.h>
void test1(int fd)
{
char buf[8192] __attribute__((aligned(8192)));
size_t count = 4096;
ssize_t ret;
while (count > 0) {
ret = write(fd, buf, count);
if (ret < 0) {
printf("write %d bytes failed. ret:%d, %s\n",
count, ret, strerror(errno));
break;
}
count /= 2;
}
return;
}
int test2(int fd)
{
char buf[8192] __attribute__((aligned(8192)));
size_t count = 4096;
ssize_t ret;
while (count > 0) {
ret = write(fd, buf + count, 512);
if (ret < 0) {
printf("write offset %d failed. ret:%d, %s\n",
count, ret, strerror(errno));
break;
}
count /= 2;
}
return;
}
int main()
{
int fd = open("testfile", O_RDWR | O_CREAT | O_DIRECT, S_IRWXU | S_IRWXG | S_IRWXO);
if (fd < 0) {
printf("open file failed.\n");
return -1;
}
printf("1. test count.\n");
test1(fd);
printf("2. test offset.\n");
test2(fd);
close(fd);
return 0;
}
通过ioctl BLKSSZGET获取设备的block大小
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/ioctl.h>
#include <linux/fs.h>
#include <string.h>
#include <errno.h>
int main(int argc, char* argv[])
{
int fd, rc;
size_t bs;
if (argc != 2) {
printf("ioctl /dev/sad.\n");
return -1;
}
fd = open(argv[1], O_RDONLY);
if (fd < 0) {
printf("open %s failed! ret %d, %s\n",
argv[1], fd, strerror(errno));
return -1;
}
rc = ioctl(fd, BLKSSZGET, &bs);
if (rc < 0) {
printf("open %s failed! ret %d, %s\n",
argv[1], rc, strerror(errno));
} else {
printf("dev %s bs is %d\n", argv[1], bs);
}
close(fd);
return 0;
}
