From 45e2ef0a8997cae84db2cf218ad2f46370972dad Mon Sep 17 00:00:00 2001 From: Numan Siddique Date: Wed, 18 Mar 2015 15:42:02 +0530 Subject: [PATCH] Update the collector column families from Ascii to UTF8 Cassandra returns InvalidRequestException when collector tries to add a column and if the column data contains unicode characters. The issue is seen when any resource is created or deleted and the resource name has unicode characters. This patch fixes the issue by changing the collector column family validation types from AsciiType to UTF8Type. The patch also takes care of upgrading the existing column families if they exist in the db. (cherry-picked from commit 358765ed4e89430414284185f0acd4f149aeba73) Change-Id: I54ae9e756f7f67ce684ef9e3ba557038bc8f8731 Closes-bug: #1433021 --- src/analytics/db_handler.cc | 1 + src/analytics/vizd_table_desc.cc | 166 +++++++++++++++---------------- src/gendb/cdb_if.cc | 59 ++++++++++- src/gendb/cdb_if.h | 4 +- src/gendb/gendb.sandesh | 1 + src/query_engine/select.cc | 1 + 6 files changed, 143 insertions(+), 89 deletions(-) diff --git a/src/analytics/db_handler.cc b/src/analytics/db_handler.cc index 8baf0795ac8..d68168a7c98 100644 --- a/src/analytics/db_handler.cc +++ b/src/analytics/db_handler.cc @@ -1139,6 +1139,7 @@ bool FlowDataIpv4ObjectWalker::for_each(pugi::xml_node& node) { break; } case GenDb::DbDataType::AsciiType: + case GenDb::DbDataType::UTF8Type: { std::string val = node.child_value(); TXMLProtocol::unescapeXMLControlChars(val); diff --git a/src/analytics/vizd_table_desc.cc b/src/analytics/vizd_table_desc.cc index 2ce76ba4067..e1b9b7f777c 100644 --- a/src/analytics/vizd_table_desc.cc +++ b/src/analytics/vizd_table_desc.cc @@ -30,25 +30,25 @@ void init_vizd_tables() { (GenDb::DbDataType::LexicalUUIDType), boost::assign::map_list_of (g_viz_constants.SOURCE, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.NAMESPACE, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.MODULE, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.INSTANCE_ID, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.NODE_TYPE, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.CONTEXT, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.TIMESTAMP, GenDb::DbDataType::Unsigned64Type) (g_viz_constants.CATEGORY, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.LEVEL, GenDb::DbDataType::Unsigned32Type) (g_viz_constants.MESSAGE_TYPE, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.SEQUENCE_NUM, GenDb::DbDataType::Unsigned32Type) (g_viz_constants.VERSION, @@ -56,15 +56,15 @@ void init_vizd_tables() { (g_viz_constants.SANDESH_TYPE, GenDb::DbDataType::Unsigned8Type) (g_viz_constants.IPADDRESS, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.PID, GenDb::DbDataType::Unsigned32Type) (g_viz_constants.DATA, - GenDb::DbDataType::AsciiType))) + GenDb::DbDataType::UTF8Type))) (GenDb::NewCf(g_viz_constants.MESSAGE_TABLE_SOURCE, boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) - (GenDb::DbDataType::AsciiType), + (GenDb::DbDataType::UTF8Type), boost::assign::list_of (GenDb::DbDataType::Unsigned32Type), boost::assign::list_of @@ -72,7 +72,7 @@ void init_vizd_tables() { (GenDb::NewCf(g_viz_constants.MESSAGE_TABLE_MODULE_ID, boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) - (GenDb::DbDataType::AsciiType), + (GenDb::DbDataType::UTF8Type), boost::assign::list_of (GenDb::DbDataType::Unsigned32Type), boost::assign::list_of @@ -80,7 +80,7 @@ void init_vizd_tables() { (GenDb::NewCf(g_viz_constants.MESSAGE_TABLE_MESSAGE_TYPE, boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) - (GenDb::DbDataType::AsciiType), + (GenDb::DbDataType::UTF8Type), boost::assign::list_of (GenDb::DbDataType::Unsigned32Type), boost::assign::list_of @@ -88,7 +88,7 @@ void init_vizd_tables() { (GenDb::NewCf(g_viz_constants.MESSAGE_TABLE_CATEGORY, boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) - (GenDb::DbDataType::AsciiType), + (GenDb::DbDataType::UTF8Type), boost::assign::list_of (GenDb::DbDataType::Unsigned32Type), boost::assign::list_of @@ -103,14 +103,14 @@ void init_vizd_tables() { (GenDb::NewCf(g_viz_constants.OBJECT_VALUE_TABLE, boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) - (GenDb::DbDataType::AsciiType), + (GenDb::DbDataType::UTF8Type), boost::assign::list_of (GenDb::DbDataType::Unsigned32Type), boost::assign::list_of - (GenDb::DbDataType::AsciiType))) + (GenDb::DbDataType::UTF8Type))) (GenDb::NewCf(g_viz_constants.SYSTEM_OBJECT_TABLE, boost::assign::list_of - (GenDb::DbDataType::AsciiType), + (GenDb::DbDataType::UTF8Type), boost::assign::map_list_of (g_viz_constants.SYSTEM_OBJECT_START_TIME, GenDb::DbDataType::Unsigned64Type))) @@ -118,16 +118,16 @@ void init_vizd_tables() { boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned8Type) - (GenDb::DbDataType::AsciiType), + (GenDb::DbDataType::UTF8Type), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type), boost::assign::list_of (GenDb::DbDataType::LexicalUUIDType))) (GenDb::NewCf(g_viz_constants.MESSAGE_TABLE_KEYWORD, boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) - (GenDb::DbDataType::AsciiType), + (GenDb::DbDataType::UTF8Type), boost::assign::list_of (GenDb::DbDataType::Unsigned32Type), boost::assign::list_of @@ -166,15 +166,15 @@ void init_vizd_tables() { (GenDb::DbDataType::LexicalUUIDType), boost::assign::map_list_of (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_VROUTER], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DIRECTION_ING], GenDb::DbDataType::Unsigned8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_SOURCEVN], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_SOURCEIP], GenDb::DbDataType::Unsigned32Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DESTVN], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DESTIP], GenDb::DbDataType::Unsigned32Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_PROTOCOL], @@ -188,15 +188,15 @@ void init_vizd_tables() { (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_TCP_FLAGS], GenDb::DbDataType::Unsigned8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_VM], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_INPUT_INTERFACE], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_OUTPUT_INTERFACE], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_MPLS_LABEL], GenDb::DbDataType::Unsigned32Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_REVERSE_UUID], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_SETUP_TIME], GenDb::DbDataType::Unsigned64Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_TEARDOWN_TIME], @@ -214,17 +214,17 @@ void init_vizd_tables() { (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_PACKETS], GenDb::DbDataType::Unsigned64Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DATA_SAMPLE], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_ACTION], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_SG_RULE_UUID], GenDb::DbDataType::LexicalUUIDType) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_NW_ACE_UUID], GenDb::DbDataType::LexicalUUIDType) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_VROUTER_IP], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_OTHER_VROUTER_IP], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_UNDERLAY_PROTO], GenDb::DbDataType::Unsigned16Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_UNDERLAY_SPORT], @@ -238,7 +238,7 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::Unsigned8Type), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), @@ -247,15 +247,15 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned64Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::LexicalUUIDType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::Unsigned16Type) (GenDb::DbDataType::Unsigned16Type) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* (DVN,DIP) index table */ @@ -265,7 +265,7 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::Unsigned8Type), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), @@ -274,15 +274,15 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned64Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::LexicalUUIDType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::Unsigned16Type) (GenDb::DbDataType::Unsigned16Type) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* (PROT, SP) index table */ @@ -301,15 +301,15 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned64Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::LexicalUUIDType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::Unsigned16Type) (GenDb::DbDataType::Unsigned16Type) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* (PROT, DP) index table */ (GenDb::NewCf(g_viz_constants.FLOW_TABLE_PROT_DP, @@ -327,15 +327,15 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned64Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::LexicalUUIDType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::Unsigned16Type) (GenDb::DbDataType::Unsigned16Type) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* (VROUTER) index table */ (GenDb::NewCf(g_viz_constants.FLOW_TABLE_VROUTER, @@ -344,7 +344,7 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::Unsigned8Type), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), boost::assign::list_of @@ -352,15 +352,15 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned64Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::LexicalUUIDType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::Unsigned16Type) (GenDb::DbDataType::Unsigned16Type) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) ; @@ -370,9 +370,9 @@ void init_vizd_tables() { GenDb::DbDataTypeVec stat_row_onetag = boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned8Type) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) ; /* For Stat Tables that have two tags @@ -381,10 +381,10 @@ void init_vizd_tables() { GenDb::DbDataTypeVec stat_row_twotag = boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned8Type) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) ; vizd_stat_tables = @@ -397,12 +397,12 @@ void init_vizd_tables() { (GenDb::NewCf(g_viz_constants.STATS_TABLE_BY_STR_STR_TAG, stat_row_twotag, boost::assign::list_of - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* Stats Table by String,U64 tag * The schema is as follows: @@ -412,12 +412,12 @@ void init_vizd_tables() { (GenDb::NewCf(g_viz_constants.STATS_TABLE_BY_STR_U64_TAG, stat_row_twotag, boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned64Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* Stats Table by U64,Str tag * The schema is as follows: @@ -428,11 +428,11 @@ void init_vizd_tables() { stat_row_twotag, boost::assign::list_of (GenDb::DbDataType::Unsigned64Type) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* Stats Table by U64,U64 tag * The schema is as follows: @@ -447,7 +447,7 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* Stats Table by U64 tag * The schema is as follows: @@ -461,7 +461,7 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* Stats Table by Double tag * The schema is as follows: @@ -475,7 +475,7 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* Stats Table by String tag * The schema is as follows: @@ -485,11 +485,11 @@ void init_vizd_tables() { (GenDb::NewCf(g_viz_constants.STATS_TABLE_BY_STR_TAG, stat_row_onetag, boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) ; @@ -498,11 +498,11 @@ void init_vizd_tables() { flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DIRECTION_ING]] = FlowTypeInfo(FlowRecordFields::FLOWREC_DIRECTION_ING, GenDb::DbDataType::Unsigned8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_SOURCEVN]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_SOURCEVN, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_SOURCEVN, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_SOURCEIP]] = FlowTypeInfo(FlowRecordFields::FLOWREC_SOURCEIP, GenDb::DbDataType::Unsigned32Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DESTVN]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_DESTVN, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_DESTVN, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DESTIP]] = FlowTypeInfo(FlowRecordFields::FLOWREC_DESTIP, GenDb::DbDataType::Unsigned32Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_PROTOCOL]] = @@ -516,15 +516,15 @@ void init_vizd_tables() { flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_TCP_FLAGS]] = FlowTypeInfo(FlowRecordFields::FLOWREC_TCP_FLAGS, GenDb::DbDataType::Unsigned8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_VM]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_VM, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_VM, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_INPUT_INTERFACE]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_INPUT_INTERFACE, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_INPUT_INTERFACE, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_OUTPUT_INTERFACE]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_OUTPUT_INTERFACE, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_OUTPUT_INTERFACE, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_MPLS_LABEL]] = FlowTypeInfo(FlowRecordFields::FLOWREC_MPLS_LABEL, GenDb::DbDataType::Unsigned32Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_REVERSE_UUID]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_REVERSE_UUID, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_REVERSE_UUID, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_SETUP_TIME]] = FlowTypeInfo(FlowRecordFields::FLOWREC_SETUP_TIME, GenDb::DbDataType::Unsigned64Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_TEARDOWN_TIME]] = @@ -546,17 +546,17 @@ void init_vizd_tables() { flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DIFF_PACKETS]] = FlowTypeInfo(FlowRecordFields::FLOWREC_DIFF_PACKETS, GenDb::DbDataType::Unsigned64Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DATA_SAMPLE]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_DATA_SAMPLE, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_DATA_SAMPLE, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_ACTION]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_ACTION, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_ACTION, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_SG_RULE_UUID]] = FlowTypeInfo(FlowRecordFields::FLOWREC_SG_RULE_UUID, GenDb::DbDataType::LexicalUUIDType); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_NW_ACE_UUID]] = FlowTypeInfo(FlowRecordFields::FLOWREC_NW_ACE_UUID, GenDb::DbDataType::LexicalUUIDType); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_VROUTER_IP]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_VROUTER_IP, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_VROUTER_IP, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_OTHER_VROUTER_IP]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_OTHER_VROUTER_IP, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_OTHER_VROUTER_IP, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_UNDERLAY_PROTO]] = FlowTypeInfo(FlowRecordFields::FLOWREC_UNDERLAY_PROTO, GenDb::DbDataType::Unsigned16Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_UNDERLAY_SPORT]] = diff --git a/src/gendb/cdb_if.cc b/src/gendb/cdb_if.cc index ea2bd21a8dc..8358e284acb 100644 --- a/src/gendb/cdb_if.cc +++ b/src/gendb/cdb_if.cc @@ -543,7 +543,13 @@ CdbIf::CdbIfTypeMapDef CdbIf::CdbIfTypeMap = DbEncodeDoubleComposite, DbDecodeDoubleComposite, DbEncodeDoubleNonComposite, - DbDecodeDoubleNonComposite)); + DbDecodeDoubleNonComposite)) + (GenDb::DbDataType::UTF8Type, + CdbIf::CdbIfTypeInfo("UTF8Type", + DbEncodeStringComposite, + DbDecodeStringComposite, + DbEncodeStringNonComposite, + DbDecodeStringNonComposite)); class CdbIf::CleanupTask : public Task { public: @@ -903,7 +909,7 @@ bool CdbIf::DbDataTypeVecToCompositeType(std::string& res, res = it->second.cassandra_type_; return true; } else { - res = "CompositeType("; + res = "org.apache.cassandra.db.marshal.CompositeType("; std::vector::const_iterator it = db_type.begin(); CdbIfTypeMapDef::iterator jt; @@ -913,7 +919,7 @@ bool CdbIf::DbDataTypeVecToCompositeType(std::string& res, it++; for (; it != db_type.end(); it++) { - res.append(", "); + res.append(","); if ((jt = CdbIfTypeMap.find(*it)) == CdbIfTypeMap.end()) return false; res.append(jt->second.cassandra_type_); @@ -1127,7 +1133,8 @@ bool CdbIf::Db_AddColumnfamily(const GenDb::NewCf& cf) { CdbIfCfInfo *cfinfo; if (Db_GetColumnfamily(&cfinfo, cf.cfname_)) { - if ((*cfinfo->cfdef_.get()).gc_grace_seconds != 0) { + if ((*cfinfo->cfdef_.get()).gc_grace_seconds != 0 || + DB_IsCfSchemaChanged(cfinfo->cfdef_.get(), &cf_def)) { CDBIF_LOG(DEBUG, "CFName: " << cf.cfname_ << " ID: " << (*cfinfo->cfdef_.get()).id); cf_def.__set_id((*cfinfo->cfdef_.get()).id); @@ -1189,7 +1196,8 @@ bool CdbIf::Db_AddColumnfamily(const GenDb::NewCf& cf) { CdbIfCfInfo *cfinfo; if (Db_GetColumnfamily(&cfinfo, cf.cfname_)) { - if ((*cfinfo->cfdef_.get()).gc_grace_seconds != 0) { + if (((*cfinfo->cfdef_.get()).gc_grace_seconds != 0) || + DB_IsCfSchemaChanged(cfinfo->cfdef_.get(), &cf_def)) { CDBIF_LOG(DEBUG, "CFName: " << cf.cfname_ << " ID: " << (*cfinfo->cfdef_.get()).id); cf_def.__set_id((*cfinfo->cfdef_.get()).id); @@ -1222,6 +1230,47 @@ bool CdbIf::Db_AddColumnfamily(const GenDb::NewCf& cf) { return true; } +bool CdbIf::DB_IsCfSchemaChanged(org::apache::cassandra::CfDef *cfdef, + org::apache::cassandra::CfDef *newcfdef) { + if (cfdef->key_validation_class != newcfdef->key_validation_class) { + return true; + } + + if ((newcfdef->default_validation_class.size()) && + (cfdef->default_validation_class != newcfdef->default_validation_class)) { + return true; + } + + if (cfdef->comparator_type != newcfdef->comparator_type) { + return true; + } + + if (cfdef->column_metadata.size() != newcfdef->column_metadata.size()) { + return true; + } + + std::vector::iterator cfdef_it = cfdef->column_metadata.begin(); + std::vector::iterator newcfdef_it; + + bool schema_changed; + for(; cfdef_it != cfdef->column_metadata.end(); cfdef_it++) { + schema_changed = true; + for(newcfdef_it = newcfdef->column_metadata.begin(); + newcfdef_it != newcfdef->column_metadata.end(); newcfdef_it++) { + if(cfdef_it->name == newcfdef_it->name && + cfdef_it->validation_class == newcfdef_it->validation_class) { + schema_changed = false; + break; + } + } + if(schema_changed) { + return true; + } + } + + return false; +} + bool CdbIf::Db_AsyncAddColumn(CdbIfColList &cl) { GenDb::ColList *new_colp(cl.gendb_cl); if (new_colp == NULL) { diff --git a/src/gendb/cdb_if.h b/src/gendb/cdb_if.h index c89b1c8e1d7..05a93feec05 100644 --- a/src/gendb/cdb_if.h +++ b/src/gendb/cdb_if.h @@ -89,7 +89,7 @@ class CdbIf : public GenDb::GenDbIf { DbDecodeCompositeFunc decode_composite_fn, DbEncodeNonCompositeFunc encode_non_composite_fn, DbDecodeNonCompositeFunc decode_non_composite_fn) : - cassandra_type_(cassandra_type), + cassandra_type_("org.apache.cassandra.db.marshal." + cassandra_type), encode_composite_fn_(encode_composite_fn), decode_composite_fn_(decode_composite_fn), encode_non_composite_fn_(encode_non_composite_fn), @@ -162,6 +162,8 @@ class CdbIf : public GenDb::GenDbIf { bool Db_AsyncAddColumn(CdbIfColList &cl); bool Db_AsyncAddColumnLocked(CdbIfColList &cl); void Db_BatchAddColumn(bool done); + bool DB_IsCfSchemaChanged(org::apache::cassandra::CfDef *cfdef, + org::apache::cassandra::CfDef *newcfdef); // Read static const int kMaxQueryRows = 5000; // API to get range of column data for a range of rows diff --git a/src/gendb/gendb.sandesh b/src/gendb/gendb.sandesh index cfc8c8bb199..ba57cdd4e5a 100644 --- a/src/gendb/gendb.sandesh +++ b/src/gendb/gendb.sandesh @@ -16,6 +16,7 @@ enum DbDataType { Unsigned64Type = 7, // uint64_t DoubleType = 8, // double + UTF8Type = 9, //utf-8 string } struct DbTableInfo { diff --git a/src/query_engine/select.cc b/src/query_engine/select.cc index 04bc5a12c36..4b1ff7e1f03 100644 --- a/src/query_engine/select.cc +++ b/src/query_engine/select.cc @@ -464,6 +464,7 @@ query_status_t SelectQuery::process_query() { break; } case GenDb::DbDataType::AsciiType: + case GenDb::DbDataType::UTF8Type: { try { elem_value = boost::get(kt->second);