mmap nocache 介绍
在 Linux 系统中，使用 mmap 映射文件到内存时，可以指定不使用缓存。通过在调用 mmap 时传递 MAP_NOCACHE 标志，可以实现这一点。MAP_NOCACHE 标志告诉内核不要将映射的内存页缓存到文件系统缓存中，而是直接将内存与文件关联。这对于需要频繁读写大量数据的应用程序非常有益，因为它避免了读写数据时的额外缓存开销。
示例代码：
#include <sys/mman.h>
#include <fcntl.h>
#include <unistd.h>
int main() {
int fd = open("file.txt", O_RDWR);
char *ptr = mmap(NULL, 100, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_NOCACHE, fd, 0);
// 使用 ptr 直接从文件中读取和写入数据
...
munmap(ptr, 100);
close(fd);
return 0;
}
有关 QNX 系统的更多信息，请参考 QNX 文档。
数据比较
-
memcpy cached速度：2133.333333 MB/s -
memcpy nocached速度：116.363636 MB/s -
invalidate memcpy cached速度：1333.333333 MB/s -
invalidate memcpy nocached速度：112.280702 MB/s -
asm memcpy cached速度：2133.333333 MB/s -
asm memcpy nocached速度：225.352113 MB/s
名词解释
-
mmap cached：指 mmap 使用
PROT_READ | PROT_WRITE标志。 -
mmap nocached：指 mmap 使用
PROT_READ | PROT_WRITE | PROT_NOCACHE标志。
memcpy
指单纯调用 memcpy，例如：
for (size_t i = 0; i < count; i++) {
memcpy(dst, src[i], bytes);
}
invalidate memcpy
指在 memcpy 前先用 msync 设置共享内存属性，例如：
for (size_t i = 0; i < count; i++) {
msync(src[i], bytes, MS_INVALIDATE);
memcpy(dst, src[i], bytes);
}
asm memcpy
指汇编版本的 memcpy，例如：
inline void aarch64_fast_memcpy(void *dst, const void *src, size_t size) {
#ifdef _QNX_
void *ss = (void *)src, *dd = (void *)dst;
size_t sz = size;
asm volatile("loop_start: "
"ldp q3, q4,[%0,#0x0]\n"
"ldp q5, q6, [%0,#0x20]\n"
"ldp q7, q8, [%0,#0x40]\n"
"ldp q9, q10, [%0,#0x60]\n"
"stp q3, q4, [%1,#0x0]\n"
"stp q5, q6, [%1,#0x20]\n"
"stp q7, q8, [%1,#0x40]\n"
"stp q9, q10, [%1,#0x60]\n"
"add %0, %0, #0x80\n"
"add %1, %1, #0x80\n"
"subs %2, %2, #0x80\n"
"b.ne loop_start\n"
"dsb sy\n"
: /* no output */
: "r"(ss), "r"(dd), "r"(sz));
#endif
}
// ......
for (size_t i = 0; i < count; i++) {
aarch64_fast_memcpy(dst, src[i], bytes);
}
（完整代码见后面的长源码文件）
结论
-
缓存情况下：
memcpy和asm memcpy速度相近，没有明显区别，invalidate memcpy最慢。 -
非缓存情况下：
asm memcpy最快，memcpy次之，invalidate memcpy最慢。
测试源码
// mmap_memcpy.c
#include <errno.h>
#include <fcntl.h>
#include <memory.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#ifndef _QNX_
#define PROT_NOCACHE 0
#endif
inline void aarch64_fast_memcpy(void *dst, const void *src, size_t size) {
#ifdef _QNX_
void *ss = (void *)src, *dd = (void *)dst;
size_t sz = size;
asm volatile("loop_start: "
"ldp q3, q4,[%0,#0x0]\n"
"ldp q5, q6, [%0,#0x20]\n"
"ldp q7, q8, [%0,#0x40]\n"
"ldp q9, q10, [%0,#0x60]\n"
"stp q3, q4, [%1,#0x0]\n"
"stp q5, q6, [%1,#0x20]\n"
"stp q7, q8, [%1,#0x40]\n"
"stp q9, q10, [%1,#0x60]\n"
"add %0, %0, #0x80\n"
"add %1, %1, #0x80\n"
"subs %2, %2, #0x80\n"
"b.ne loop_start\n"
"dsb sy\n"
: /* no output */
: "r"(ss), "r"(dd), "r"(sz));
#endif
}
off_t offset(unsigned int bytes) {
static off_t base_offset = 0x1E000000;
off_t return_base_offset = base_offset;
base_offset += bytes;
return return_base_offset;
}
void *mmap_memory(unsigned int bytes, int flag) {
int fd = open("/dev/mem", O_RDWR | O_SYNC);
if (fd < 0) {
printf("open /dev/mem failed: %s\n", strerror(errno));
}
void *ptr = mmap(NULL, bytes, flag, MAP_SHARED, fd, offset(bytes));
close(fd);
if (MAP_FAILED == ptr) {
printf("mmap failed: %s\n", strerror(errno));
}
return ptr;
}
void *mmap_memory_cached(unsigned int bytes) {
return mmap_memory(bytes, PROT_READ | PROT_WRITE);
}
void *mmap_memory_nocached(unsigned int bytes) {
return mmap_memory(bytes, PROT_READ | PROT_WRITE | PROT_NOCACHE);
}
// if C++
#ifdef __cplusplus
#include <chrono>
using namespace std::chrono;
#define start() auto start_ = high_resolution_clock::now();
#define end() \
auto end_ = high_resolution_clock::now(); \
double bytes_mb = bytes * count / 1024.0 / 1024.0; \
double cost_ns = duration_cast<nanoseconds>(end_ - start_).count(); \
double mps = bytes_mb / cost_ns * 1e9;
#else
#include <time.h>
#include <unistd.h>
double time_diff_ns(struct timespec start, struct timespec end) {
return (end.tv_sec - start.tv_sec) * 1e9 + (end.tv_nsec - start.tv_nsec);
}
#define start() \
struct timespec start_, end_; \
clock_gettime(CLOCK_REALTIME, &start_);
#define end() \
clock_gettime(CLOCK_REALTIME, &end_); \
double bytes_mb = bytes * count / 1024.0 / 1024.0; \
double cost_ns = time_diff_ns(start_, end_); \
double mps = bytes_mb / cost_ns * 1e9;
#endif
double memcpy_speed(void *dst, void **src, unsigned int bytes,
unsigned int count) {
start();
for (size_t i = 0; i < count; i++) {
memcpy(dst, src[i], bytes);
}
end();
return mps;
}
double invalidate_memcpy_speed(void *dst, void **src, unsigned int bytes,
unsigned int count) {
start();
for (size_t i = 0; i < count; i++) {
msync(src[i], bytes, MS_INVALIDATE);
memcpy(dst, src[i], bytes);
}
end();
return mps;
}
double asm_memcpy_speed(void *dst, void **src, unsigned int bytes,
unsigned int count) {
start();
for (size_t i = 0; i < count; i++) {
aarch64_fast_memcpy(dst, src[i], bytes);
}
end();
return mps;
}
int main(int argc, char *argv[]) {
const unsigned int count = 2;
unsigned int bytes =
128 * 1024; // 128 KB
if (argc > 1) {
bytes = atoi(argv[1]) * 1024;
}
printf("bytes: %d\n", bytes);
printf("count: %d\n", count);
void *mmap_cached_src[count];
void *mmap_nocached_src[count];
for (size_t i = 0; i < count; i++) {
mmap_cached_src[i] = mmap_memory_cached(bytes);
mmap_nocached_src[i] = mmap_memory_nocached(bytes);
}
void *dst = malloc(bytes);
printf("memcpy cached speed: %f MB/s\n",
memcpy_speed(dst, mmap_cached_src, bytes, count));
printf("memcpy nocached speed: %f MB/s\n",
memcpy_speed(dst, mmap_nocached_src, bytes, count));
printf("invalidate memcpy cached speed: %f MB/s\n",
invalidate_memcpy_speed(dst, mmap_cached_src, bytes, count));
printf("invalidate memcpy nocached speed: %f MB/s\n",
invalidate_memcpy_speed(dst, mmap_nocached_src, bytes, count));
printf("asm memcpy cached speed: %f MB/s dst[0]=%d\n",
asm_memcpy_speed(dst, mmap_cached_src, bytes, count), ((char *)dst)[0]);
printf("asm memcpy nocached speed: %f MB/s\n",
asm_memcpy_speed(dst, mmap_nocached_src, bytes, count));
free(dst);
for (size_t i = 0; i < count; i++) {
munmap(mmap_cached_src[i], bytes);
munmap(mmap_nocached_src[i], bytes);
}
return 0;
}