Techwizz/kernel-fxtec-pro1x
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

IB/iser: make FMR "page size" be 4K and not PAGE_SIZE

Browse source 
As iser is able to use at most one rdma operation for the
execution of a scsi command, and registration of the sg
associated with scsi command has its restrictions, the code
checks if an sg is "aligned for rdma".

Alignment for rdma is measured in "fmr page" units whose
possible resolutions are different between HCAs and can be
smaller, equal or bigger to the system page size.

When the system page size is bigger than 4KB (eg the default
with ia64 kernels) there a bigger chance that an sg would be
aligned for rdma if the fmr page size is 4KB.

Change the code to create FMR whose pages are of size 4KB
and to take that into account when processing the sg.

Signed-off-by: Erez Zilber <erezz@voltaire.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>
This commit is contained in:
Erez Zilber 2006-09-11 12:22:30 +03:00 committed by Roland Dreier
parent 8072ec2f8f
commit 8dfa0876d3
3 changed files with 26 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
drivers/infiniband/ulp/iser/iscsi_iser.h
View file

@ -82,8 +82,12 @@
__func__ , ## arg); \ __func__ , ## arg); \
} while (0) } while (0)
#define SHIFT_4K 12
#define SIZE_4K (1UL << SHIFT_4K)
#define MASK_4K (~(SIZE_4K-1))
/* support upto 512KB in one RDMA */ /* support upto 512KB in one RDMA */
#define ISCSI_ISER_SG_TABLESIZE (0x80000 >> PAGE_SHIFT) #define ISCSI_ISER_SG_TABLESIZE (0x80000 >> SHIFT_4K)
#define ISCSI_ISER_MAX_LUN 256 #define ISCSI_ISER_MAX_LUN 256
#define ISCSI_ISER_MAX_CMD_LEN 16 #define ISCSI_ISER_MAX_CMD_LEN 16

29
drivers/infiniband/ulp/iser/iser_memory.c
View file

@ -42,6 +42,7 @@
#include "iscsi_iser.h" #include "iscsi_iser.h"
#define ISER_KMALLOC_THRESHOLD 0x20000 /* 128K - kmalloc limit */ #define ISER_KMALLOC_THRESHOLD 0x20000 /* 128K - kmalloc limit */
/** /**
* Decrements the reference count for the * Decrements the reference count for the
* registered buffer & releases it * registered buffer & releases it
@ -239,7 +240,7 @@ static int iser_sg_to_page_vec(struct iser_data_buf *data,
int i; int i;
/* compute the offset of first element */ /* compute the offset of first element */
page_vec->offset = (u64) sg[0].offset; page_vec->offset = (u64) sg[0].offset & ~MASK_4K;
for (i = 0; i < data->dma_nents; i++) { for (i = 0; i < data->dma_nents; i++) {
total_sz += sg_dma_len(&sg[i]); total_sz += sg_dma_len(&sg[i]);
@ -247,21 +248,30 @@ static int iser_sg_to_page_vec(struct iser_data_buf *data,
first_addr = sg_dma_address(&sg[i]); first_addr = sg_dma_address(&sg[i]);
last_addr = first_addr + sg_dma_len(&sg[i]); last_addr = first_addr + sg_dma_len(&sg[i]);
start_aligned = !(first_addr & ~PAGE_MASK); start_aligned = !(first_addr & ~MASK_4K);
end_aligned = !(last_addr & ~PAGE_MASK); end_aligned = !(last_addr & ~MASK_4K);
/* continue to collect page fragments till aligned or SG ends */ /* continue to collect page fragments till aligned or SG ends */
while (!end_aligned && (i + 1 < data->dma_nents)) { while (!end_aligned && (i + 1 < data->dma_nents)) {
i++; i++;
total_sz += sg_dma_len(&sg[i]); total_sz += sg_dma_len(&sg[i]);
last_addr = sg_dma_address(&sg[i]) + sg_dma_len(&sg[i]); last_addr = sg_dma_address(&sg[i]) + sg_dma_len(&sg[i]);
end_aligned = !(last_addr & ~PAGE_MASK); end_aligned = !(last_addr & ~MASK_4K);
} }
first_addr = first_addr & PAGE_MASK; /* handle the 1st page in the 1st DMA element */
if (cur_page == 0) {
page = first_addr & MASK_4K;
page_vec->pages[cur_page] = page;
cur_page++;
page += SIZE_4K;
} else
page = first_addr;
for (page = first_addr; page < last_addr; page += PAGE_SIZE) for (; page < last_addr; page += SIZE_4K) {
page_vec->pages[cur_page++] = page; page_vec->pages[cur_page] = page;
cur_page++;
}
} }
page_vec->data_size = total_sz; page_vec->data_size = total_sz;
@ -269,8 +279,7 @@ static int iser_sg_to_page_vec(struct iser_data_buf *data,
return cur_page; return cur_page;
} }
#define MASK_4K ((1UL << 12) - 1) /* 0xFFF */ #define IS_4K_ALIGNED(addr) ((((unsigned long)addr) & ~MASK_4K) == 0)
#define IS_4K_ALIGNED(addr) ((((unsigned long)addr) & MASK_4K) == 0)
/** /**
* iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned * iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned
@ -352,7 +361,7 @@ static void iser_page_vec_build(struct iser_data_buf *data,
page_vec->length = page_vec_len; page_vec->length = page_vec_len;
if (page_vec_len * PAGE_SIZE < page_vec->data_size) { if (page_vec_len * SIZE_4K < page_vec->data_size) {
iser_err("page_vec too short to hold this SG\n"); iser_err("page_vec too short to hold this SG\n");
iser_data_buf_dump(data); iser_data_buf_dump(data);
iser_dump_page_vec(page_vec); iser_dump_page_vec(page_vec);

4
drivers/infiniband/ulp/iser/iser_verbs.c
View file

@ -150,7 +150,7 @@ static int iser_create_ib_conn_res(struct iser_conn *ib_conn)
} }
ib_conn->page_vec->pages = (u64 *) (ib_conn->page_vec + 1); ib_conn->page_vec->pages = (u64 *) (ib_conn->page_vec + 1);
params.page_shift = PAGE_SHIFT; params.page_shift = SHIFT_4K;
/* when the first/last SG element are not start/end * /* when the first/last SG element are not start/end *
* page aligned, the map whould be of N+1 pages */ * page aligned, the map whould be of N+1 pages */
params.max_pages_per_fmr = ISCSI_ISER_SG_TABLESIZE + 1; params.max_pages_per_fmr = ISCSI_ISER_SG_TABLESIZE + 1;
@ -604,7 +604,7 @@ int iser_reg_page_vec(struct iser_conn *ib_conn,
mem_reg->lkey = mem->fmr->lkey; mem_reg->lkey = mem->fmr->lkey;
mem_reg->rkey = mem->fmr->rkey; mem_reg->rkey = mem->fmr->rkey;
mem_reg->len = page_vec->length * PAGE_SIZE; mem_reg->len = page_vec->length * SIZE_4K;
mem_reg->va = io_addr; mem_reg->va = io_addr;
mem_reg->mem_h = (void *)mem; mem_reg->mem_h = (void *)mem;