diff options
author | Sebastien Dugue <sebastien.dugue@bull.net> | 2009-07-29 11:45:55 -0700 |
---|---|---|
committer | Roland Dreier <rolandd@cisco.com> | 2009-07-29 11:54:28 -0700 |
commit | 87750d1db4bc73933e92d862ff73d10d06ff4f5d (patch) | |
tree | 5c9d0785df94a0f53f8430f0eb08343e33172836 | |
parent | 09f8dcd3769a52eebceeae15e2c5a844e6ee4586 (diff) | |
download | libmlx4-87750d1db4bc73933e92d862ff73d10d06ff4f5d.tar.gz |
Use mmap(MAP_ANONYMOUS) to allocate queue buffers
Internal buffers for QPs, CQs, SRQs etc. are allocated with
mlx4_alloc_buf(), which rounds the buffer's size to the page size and
then allocates page aligned memory using posix_memalign().
However, this allocation is quite wasteful on architectures using 64K
pages (ia64 for example) because we then hit glibc's MMAP_THRESHOLD
malloc parameter and chunks are allocated using mmap. Thus we end up
allocating:
(requested size rounded to the page size) + (page size) + (malloc overhead)
rounded internally to the page size.
So for example, if we request a buffer of page_size bytes, we end up
consuming 3 pages. In short, for each buffer we allocate, there is an
overhead of 2 pages. This is quite visible on large clusters where
the number of QPs can reach several thousands.
This patch replaces the call to posix_memalign() in mlx4_alloc_buf()
with a direct call to mmap().
Signed-off-by: Sebastien Dugue <sebastien.dugue@bull.net>
Signed-off-by: Roland Dreier <rolandd@cisco.com>
-rw-r--r-- | src/buf.c | 15 |
1 files changed, 7 insertions, 8 deletions
@@ -61,16 +61,15 @@ int mlx4_alloc_buf(struct mlx4_buf *buf, size_t size, int page_size) { int ret; - ret = posix_memalign(&buf->buf, page_size, align(size, page_size)); - if (ret) - return ret; + buf->length = align(size, page_size); + buf->buf = mmap(NULL, buf->length, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (buf->buf == MAP_FAILED) + return errno; ret = ibv_dontfork_range(buf->buf, size); if (ret) - free(buf->buf); - - if (!ret) - buf->length = size; + munmap(buf->buf, buf->length); return ret; } @@ -78,5 +77,5 @@ int mlx4_alloc_buf(struct mlx4_buf *buf, size_t size, int page_size) void mlx4_free_buf(struct mlx4_buf *buf) { ibv_dofork_range(buf->buf, buf->length); - free(buf->buf); + munmap(buf->buf, buf->length); } |