From c03cb7bce8dde205cb948fdb92f68616dd13aba5 Mon Sep 17 00:00:00 2001 From: Wojciech Zmuda Date: Wed, 9 Mar 2016 15:54:43 +0100 Subject: [PATCH] Flow utility: performance measurement. Based on work of Divakar Dharanalakota Add perf mode to the flow utility. Perf adds requested number of flows in HOLD state, sets them in FWD state, then invalidates and displays time in miliseconds. The perf test may be conducted only once, after start of VRouter and before any flows are created. VRouter needs to be restared before running the test again, as generation IDs in flow requests wouldn't match with VRouter's internal count. flow -p - create requested number of HOLD flows, then create the same number of FWD flows, then invalidate them. Display time in miliseconds. flow -p -b - the same as above + send requested number of netlink requests (up to 15) in a bunch. Change-Id: Ia22b643e05d7e05ce7925cb355353df4522ce6ae Closes-bug: #1555774 --- include/nl_util.h | 2 + utils/flow.c | 296 ++++++++++++++++++++++++++++++++++++++++------ utils/nl_util.c | 12 ++ 3 files changed, 275 insertions(+), 35 deletions(-) diff --git a/include/nl_util.h b/include/nl_util.h index a5e84b588..dc8e16393 100644 --- a/include/nl_util.h +++ b/include/nl_util.h @@ -50,6 +50,7 @@ struct nl_client { struct nl_response resp; unsigned int cl_resp_buf_len; uint8_t *cl_resp_buf; + uint8_t *cl_attr; int cl_socket_domain; int cl_socket_type; int cl_socket_proto; @@ -107,6 +108,7 @@ extern int nl_get_attr_hdr_size(); extern uint8_t *nl_get_buf_ptr(struct nl_client *cl); extern uint32_t nl_get_buf_len(struct nl_client *cl); extern void nl_build_attr(struct nl_client *cl, int len, int attr); +extern void nl_update_attr_len(struct nl_client *, int); extern int vrouter_get_family_id(struct nl_client *cl); extern int get_vrouter_pid(void); diff --git a/utils/flow.c b/utils/flow.c index 4cc4367b4..b754c4a34 100644 --- a/utils/flow.c +++ b/utils/flow.c @@ -46,6 +46,9 @@ #include "vr_packet.h" #define TABLE_FLAG_VALID 0x1 + +#define MAX_FLOW_NL_MSG_BUNCH 15 + #define MEM_DEV "/dev/flow" static int mem_fd; @@ -55,7 +58,8 @@ static int help_set, match_set, get_set; static unsigned short dvrf; static int list, flow_cmd, mirror = -1; static unsigned long flow_index; -static int rate, stats; +static int rate, stats, perf, flush, bunch = 1; +static bool more = false; #define FLOW_GET_FIELD_LENGTH 30 #define FLOW_COMPONENT_NH_COUNT 16 @@ -1688,44 +1692,17 @@ flow_table_map(vr_flow_req *req) static int flow_make_flow_req(vr_flow_req *req) { - int ret, attr_len, error; - struct nl_response *resp; - - ret = nl_build_nlh(cl, cl->cl_genl_family_id, NLM_F_REQUEST); - if (ret) - return ret; - - ret = nl_build_genlh(cl, SANDESH_REQUEST, 0); - if (ret) - return ret; - - attr_len = nl_get_attr_hdr_size(); - - error = 0; - ret = sandesh_encode(req, "vr_flow_req", vr_find_sandesh_info, - (nl_get_buf_ptr(cl) + attr_len), - (nl_get_buf_len(cl) - attr_len), &error); + int ret; - if ((ret <= 0) || error) { + ret = vr_sendmsg(cl, req, "vr_flow_req"); + if (ret <= 0) return ret; - } - nl_build_attr(cl, ret, NL_ATTR_VR_MESSAGE_PROTOCOL); - nl_update_nlh(cl); - ret = nl_sendmsg(cl); + ret = vr_recvmsg(cl, false); if (ret <= 0) return ret; - if ((ret = nl_recvmsg(cl)) > 0) { - resp = nl_parse_reply(cl); - if (resp->nl_op == SANDESH_REQUEST) { - sandesh_decode(resp->nl_data, resp->nl_len, vr_find_sandesh_info, &ret); - } - } - - if (errno == EAGAIN || errno == EWOULDBLOCK) - ret = 0; - + cl->cl_buf_offset = 0; return ret; } @@ -1761,6 +1738,7 @@ flow_table_setup(void) if (ret <= 0) return ret; + cl->cl_buf_offset = 0; return ret; } @@ -1770,6 +1748,7 @@ flow_do_op(unsigned long flow_index, char action) struct vr_flow_entry *fe; memset(&flow_req, 0, sizeof(flow_req)); + flow_req.fr_flow_ip_size = 8; fe = flow_get(flow_index); if (!fe) { @@ -1814,6 +1793,7 @@ flow_do_op(unsigned long flow_index, char action) flow_req.fr_flow_sport = fe->fe_key.flow_sport; flow_req.fr_flow_dport = fe->fe_key.flow_dport; flow_req.fr_flow_nh_id = fe->fe_key.flow_nh_id; + flow_req.fr_gen_id = fe->fe_gen_id; switch (action) { case 'd': @@ -1852,6 +1832,225 @@ flow_do_op(unsigned long flow_index, char action) return; } +static int +flow_make_flow_req_perf(vr_flow_req *req) +{ + int ret, attr_len, error; + struct nl_response *resp; + static count = 0; + static struct iovec iov[MAX_FLOW_NL_MSG_BUNCH]; + uint8_t *base; + + base = nl_get_buf_ptr(cl); + + if (!count) { + ret = nl_build_nlh(cl, cl->cl_genl_family_id, NLM_F_REQUEST); + if (ret) + return ret; + + ret = nl_build_genlh(cl, SANDESH_REQUEST, 0); + if (ret) + return ret; + + attr_len = nl_get_attr_hdr_size(); + } else { + attr_len = 0; + } + + error = 0; + ret = sandesh_encode(req, "vr_flow_req", vr_find_sandesh_info, + (nl_get_buf_ptr(cl) + attr_len), + (nl_get_buf_len(cl) - attr_len), &error); + + if ((ret <= 0) || error) + return ret; + + if (!count) { + nl_build_attr(cl, ret, NL_ATTR_VR_MESSAGE_PROTOCOL); + } else { + nl_update_attr_len(cl, ret); + } + + nl_update_nlh(cl); + + iov[count].iov_base = base; + iov[count].iov_len = nl_get_buf_ptr(cl) - base; + count++; + if (bunch != count && more) + return 0; + + struct msghdr msg; + memset(&msg, 0, sizeof(msg)); +#if defined (__linux__) + msg.msg_name = cl->cl_sa; + msg.msg_namelen = cl->cl_sa_len; +#endif + + msg.msg_iov = iov; + msg.msg_iovlen = count; + + ret = sendmsg(cl->cl_sock, &msg, 0); + if (ret <= 0) + return ret; + + cl->cl_buf_offset = 0; + count = 0; + if (errno == EAGAIN || errno == EWOULDBLOCK) + ret = 0; + + return ret; +} + +void run_perf(void) { + struct vr_flow_entry *fe; + struct nl_response *resp; + uint32_t sip = inet_addr("1.1.1.1"); + uint32_t dip = inet_addr("2.2.2.2"); + int ret; + uint8_t proto = 0xFF; + uint16_t sport = 1000; + uint16_t nhid = 1; + + memset(&flow_req, 0, sizeof(flow_req)); + flow_req.fr_family = AF_INET; + flow_req.fr_flow_ip = malloc(8); + if (!flow_req.fr_flow_ip) + return; + flow_req.fr_flow_ip_size = 8; + flow_req.fr_op = FLOW_OP_FLOW_SET; + flow_req.fr_index = -1; + flow_req.fr_flags = VR_FLOW_FLAG_ACTIVE; + memcpy(flow_req.fr_flow_ip, (uint8_t *)&sip, sizeof(sip)); + memcpy(flow_req.fr_flow_ip + 4, (uint8_t *)&dip, sizeof(dip)); + flow_req.fr_flow_proto = proto; + flow_req.fr_flow_sport = htons(sport); + flow_req.fr_flow_nh_id = nhid; + flow_req.fr_gen_id = 1; + + struct timeval last_time; + gettimeofday(&last_time, NULL); + + int i = 0; + for (i = 0; i < perf; i++) { + flow_req.fr_action = VR_FLOW_ACTION_HOLD; + flow_req.fr_flow_dport = htons(i); + flow_make_flow_req_perf(&flow_req); + } + + /* process responses from previous requests */ + while (i != 0) { + i--; + cl->cl_buf_offset = 0; + if ((ret = nl_recvmsg(cl)) > 0) { + resp = nl_parse_reply(cl); + if (resp->nl_op == SANDESH_REQUEST) { + sandesh_decode(resp->nl_data, resp->nl_len, + vr_find_sandesh_info, &ret); + } + } + } + cl->cl_buf_offset = 0; + + struct timeval now; + gettimeofday(&now, NULL); + int diff_ms; + diff_ms = (now.tv_sec - last_time.tv_sec) * 1000; + diff_ms += (now.tv_usec - last_time.tv_usec) / 1000; + printf("Created %d HOLD entries in %d msec\n", perf, diff_ms); + + int flow_index[perf]; + for (i = 0; i < perf; i++) + flow_index[i] = -1; + struct flow_table *ft = &main_table; + for (i = 0; i < ft->ft_num_entries; i++) { + fe = flow_get(i); + if (fe->fe_type != VP_TYPE_IP) + continue; + if (fe->fe_key.flow4_proto != proto) + continue; + + flow_index[ntohs(fe->fe_key.flow4_dport)] = i; + } + + gettimeofday(&last_time, NULL); + + for (i = 0; i < perf; i++) { + memcpy(flow_req.fr_flow_ip, (uint8_t *)&dip, sizeof(dip)); + memcpy(flow_req.fr_flow_ip + 4, (uint8_t *)&sip, sizeof(sip)); + flow_req.fr_flow_sport = htons(i); + flow_req.fr_flow_dport = htons(sport); + flow_req.fr_index = -1; + flow_req.fr_action = VR_FLOW_ACTION_FORWARD; + more = true; + flow_make_flow_req_perf(&flow_req); + + memcpy(flow_req.fr_flow_ip, (uint8_t *)&sip, sizeof(sip)); + memcpy(flow_req.fr_flow_ip + 4, (uint8_t *)&dip, sizeof(dip)); + flow_req.fr_flow_sport = htons(sport); + flow_req.fr_flow_dport = htons(i); + flow_req.fr_index = flow_index[i]; + flow_req.fr_action = VR_FLOW_ACTION_FORWARD; + if (i == (perf - 1)) { + more = false; + } + flow_make_flow_req_perf(&flow_req); + } + + /* process responses from previous requests */ + i *= 2; /* there were two requests per one loop cycle */ + while (i != 0) { + i--; + cl->cl_buf_offset = 0; + if ((ret = nl_recvmsg(cl)) > 0) { + resp = nl_parse_reply(cl); + if (resp->nl_op == SANDESH_REQUEST) { + sandesh_decode(resp->nl_data, resp->nl_len, + vr_find_sandesh_info, &ret); + } + } + } + cl->cl_buf_offset = 0; + + gettimeofday(&now, NULL); + diff_ms = (now.tv_sec - last_time.tv_sec) * 1000; + diff_ms += (now.tv_usec - last_time.tv_usec) / 1000; + printf("Created %d HOLD and %d FWD entries in %d msec\n", + perf, perf, diff_ms); + + free(flow_req.fr_flow_ip); + + for (i = 0; i < ft->ft_num_entries; i++) { + fe = flow_get(i); + if (fe->fe_type != VP_TYPE_IP) + continue; + flow_do_op(i, 'i'); + } +} + +void run_flush(void) { + struct vr_flow_entry *fe; + struct flow_table *ft = &main_table; + struct timeval now; + struct timeval last_time; + int diff_ms; + + gettimeofday(&last_time, NULL); + int i; + int count = 0; + for (i = 0; i < ft->ft_num_entries; i++) { + fe = flow_get(i); + if (fe->fe_type != VP_TYPE_IP) + continue; + count++; + flow_do_op(i, 'i'); + } + + gettimeofday(&now, NULL); + diff_ms = (now.tv_sec - last_time.tv_sec) * 1000; + diff_ms += (now.tv_usec - last_time.tv_usec) / 1000; + printf("Deleted %d entries in %d msec\n", count, diff_ms); +} + static void Usage(void) { @@ -1911,7 +2110,8 @@ static struct option long_options[] = { static void validate_options(void) { - if (!flow_index && !list && !rate && !stats && !match_set) + if (!flow_index && !list && !rate && !stats && !match_set + && !perf && !flush) Usage(); if (show_evicted_set && !list) @@ -2208,7 +2408,7 @@ main(int argc, char *argv[]) int ret; int option_index; - while ((opt = getopt_long(argc, argv, "d:f:g:i:lrs", + while ((opt = getopt_long(argc, argv, "d:f:g:i:p:b:lrsF", long_options, &option_index)) >= 0) { switch (opt) { case 'f': @@ -2226,9 +2426,31 @@ main(int argc, char *argv[]) case 'r': rate = 1; break; + case 's': stats = 1; break; + + case 'F': + flush = 1; + break; + + case 'p': + perf = strtoul(optarg, NULL, 0); + break; + + case 'b' : + bunch = strtoul(optarg, NULL, 0); + if (bunch < 1) { + bunch = 1; + } + if (bunch > MAX_FLOW_NL_MSG_BUNCH) { + printf("Max NETLINK messages in a bunch cannot exceed %u.\n", + MAX_FLOW_NL_MSG_BUNCH); + bunch = MAX_FLOW_NL_MSG_BUNCH; + } + break; + case 0: parse_long_opts(option_index, optarg); break; @@ -2254,6 +2476,10 @@ main(int argc, char *argv[]) flow_rate(); } else if (stats) { flow_stats(); + } else if (perf) { + run_perf(); + } else if (flush) { + run_flush(); } else { if (flow_index >= main_table.ft_num_entries) { printf("Flow index %lu is greater than available indices (%u)\n", diff --git a/utils/nl_util.c b/utils/nl_util.c index d4222baaa..f3ee1610f 100644 --- a/utils/nl_util.c +++ b/utils/nl_util.c @@ -384,12 +384,24 @@ nl_get_buf_len(struct nl_client *cl) return cl->cl_buf_len - cl->cl_buf_offset; } +void +nl_update_attr_len(struct nl_client *cl, int len) +{ + struct nlattr *nla; + + nla = (struct nlattr *)cl->cl_attr; + nla->nla_len += len; + cl->cl_buf_offset += len; + return; +} + void nl_build_attr(struct nl_client *cl, int len, int attr) { struct nlattr *nla; nla = (struct nlattr *)(cl->cl_buf + cl->cl_buf_offset); + cl->cl_attr = (uint8_t *)nla; nla->nla_len = NLA_HDRLEN + (len); nla->nla_type = attr;