From 358765ed4e89430414284185f0acd4f149aeba73 Mon Sep 17 00:00:00 2001 From: Numan Siddique Date: Wed, 18 Mar 2015 15:42:02 +0530 Subject: [PATCH] Update the collector column families from Ascii to UTF8 Cassandra returns InvalidRequestException when collector tries to add a column and if the column data contains unicode characters. The issue is seen when any resource is created or deleted and the resource name has unicode characters. This patch fixes the issue by changing the collector column family validation types from AsciiType to UTF8Type. The patch also takes care of upgrading the existing column families if they exist in the db. Change-Id: I54ae9e756f7f67ce684ef9e3ba557038bc8f8731 Closes-bug: #1433021 --- src/analytics/db_handler.cc | 1 + src/analytics/vizd_table_desc.cc | 166 +++++++++++++++---------------- src/gendb/cdb_if.cc | 59 ++++++++++- src/gendb/cdb_if.h | 4 +- src/gendb/gendb.sandesh | 1 + src/query_engine/select.cc | 1 + 6 files changed, 143 insertions(+), 89 deletions(-) diff --git a/src/analytics/db_handler.cc b/src/analytics/db_handler.cc index 87813febf6d..576e285ec6c 100644 --- a/src/analytics/db_handler.cc +++ b/src/analytics/db_handler.cc @@ -1192,6 +1192,7 @@ bool FlowDataIpv4ObjectWalker::for_each(pugi::xml_node& node) { break; } case GenDb::DbDataType::AsciiType: + case GenDb::DbDataType::UTF8Type: { std::string val = node.child_value(); TXMLProtocol::unescapeXMLControlChars(val); diff --git a/src/analytics/vizd_table_desc.cc b/src/analytics/vizd_table_desc.cc index 036cbffedda..6e3d032bd1b 100644 --- a/src/analytics/vizd_table_desc.cc +++ b/src/analytics/vizd_table_desc.cc @@ -30,25 +30,25 @@ void init_vizd_tables() { (GenDb::DbDataType::LexicalUUIDType), boost::assign::map_list_of (g_viz_constants.SOURCE, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.NAMESPACE, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.MODULE, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.INSTANCE_ID, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.NODE_TYPE, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.CONTEXT, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.TIMESTAMP, GenDb::DbDataType::Unsigned64Type) (g_viz_constants.CATEGORY, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.LEVEL, GenDb::DbDataType::Unsigned32Type) (g_viz_constants.MESSAGE_TYPE, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.SEQUENCE_NUM, GenDb::DbDataType::Unsigned32Type) (g_viz_constants.VERSION, @@ -56,15 +56,15 @@ void init_vizd_tables() { (g_viz_constants.SANDESH_TYPE, GenDb::DbDataType::Unsigned8Type) (g_viz_constants.IPADDRESS, - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.PID, GenDb::DbDataType::Unsigned32Type) (g_viz_constants.DATA, - GenDb::DbDataType::AsciiType))) + GenDb::DbDataType::UTF8Type))) (GenDb::NewCf(g_viz_constants.MESSAGE_TABLE_SOURCE, boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) - (GenDb::DbDataType::AsciiType), + (GenDb::DbDataType::UTF8Type), boost::assign::list_of (GenDb::DbDataType::Unsigned32Type), boost::assign::list_of @@ -72,7 +72,7 @@ void init_vizd_tables() { (GenDb::NewCf(g_viz_constants.MESSAGE_TABLE_MODULE_ID, boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) - (GenDb::DbDataType::AsciiType), + (GenDb::DbDataType::UTF8Type), boost::assign::list_of (GenDb::DbDataType::Unsigned32Type), boost::assign::list_of @@ -80,7 +80,7 @@ void init_vizd_tables() { (GenDb::NewCf(g_viz_constants.MESSAGE_TABLE_MESSAGE_TYPE, boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) - (GenDb::DbDataType::AsciiType), + (GenDb::DbDataType::UTF8Type), boost::assign::list_of (GenDb::DbDataType::Unsigned32Type), boost::assign::list_of @@ -88,7 +88,7 @@ void init_vizd_tables() { (GenDb::NewCf(g_viz_constants.MESSAGE_TABLE_CATEGORY, boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) - (GenDb::DbDataType::AsciiType), + (GenDb::DbDataType::UTF8Type), boost::assign::list_of (GenDb::DbDataType::Unsigned32Type), boost::assign::list_of @@ -103,14 +103,14 @@ void init_vizd_tables() { (GenDb::NewCf(g_viz_constants.OBJECT_VALUE_TABLE, boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) - (GenDb::DbDataType::AsciiType), + (GenDb::DbDataType::UTF8Type), boost::assign::list_of (GenDb::DbDataType::Unsigned32Type), boost::assign::list_of - (GenDb::DbDataType::AsciiType))) + (GenDb::DbDataType::UTF8Type))) (GenDb::NewCf(g_viz_constants.SYSTEM_OBJECT_TABLE, boost::assign::list_of - (GenDb::DbDataType::AsciiType), + (GenDb::DbDataType::UTF8Type), boost::assign::map_list_of (g_viz_constants.SYSTEM_OBJECT_START_TIME, GenDb::DbDataType::Unsigned64Type) @@ -125,16 +125,16 @@ void init_vizd_tables() { boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned8Type) - (GenDb::DbDataType::AsciiType), + (GenDb::DbDataType::UTF8Type), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type), boost::assign::list_of (GenDb::DbDataType::LexicalUUIDType))) (GenDb::NewCf(g_viz_constants.MESSAGE_TABLE_KEYWORD, boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) - (GenDb::DbDataType::AsciiType), + (GenDb::DbDataType::UTF8Type), boost::assign::list_of (GenDb::DbDataType::Unsigned32Type), boost::assign::list_of @@ -173,15 +173,15 @@ void init_vizd_tables() { (GenDb::DbDataType::LexicalUUIDType), boost::assign::map_list_of (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_VROUTER], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DIRECTION_ING], GenDb::DbDataType::Unsigned8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_SOURCEVN], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_SOURCEIP], GenDb::DbDataType::Unsigned32Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DESTVN], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DESTIP], GenDb::DbDataType::Unsigned32Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_PROTOCOL], @@ -195,15 +195,15 @@ void init_vizd_tables() { (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_TCP_FLAGS], GenDb::DbDataType::Unsigned8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_VM], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_INPUT_INTERFACE], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_OUTPUT_INTERFACE], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_MPLS_LABEL], GenDb::DbDataType::Unsigned32Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_REVERSE_UUID], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_SETUP_TIME], GenDb::DbDataType::Unsigned64Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_TEARDOWN_TIME], @@ -221,17 +221,17 @@ void init_vizd_tables() { (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_PACKETS], GenDb::DbDataType::Unsigned64Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DATA_SAMPLE], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_ACTION], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_SG_RULE_UUID], GenDb::DbDataType::LexicalUUIDType) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_NW_ACE_UUID], GenDb::DbDataType::LexicalUUIDType) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_VROUTER_IP], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_OTHER_VROUTER_IP], - GenDb::DbDataType::AsciiType) + GenDb::DbDataType::UTF8Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_UNDERLAY_PROTO], GenDb::DbDataType::Unsigned16Type) (g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_UNDERLAY_SPORT], @@ -245,7 +245,7 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::Unsigned8Type), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), @@ -254,15 +254,15 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned64Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::LexicalUUIDType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::Unsigned16Type) (GenDb::DbDataType::Unsigned16Type) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* (DVN,DIP) index table */ @@ -272,7 +272,7 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::Unsigned8Type), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), @@ -281,15 +281,15 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned64Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::LexicalUUIDType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::Unsigned16Type) (GenDb::DbDataType::Unsigned16Type) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* (PROT, SP) index table */ @@ -308,15 +308,15 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned64Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::LexicalUUIDType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::Unsigned16Type) (GenDb::DbDataType::Unsigned16Type) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* (PROT, DP) index table */ (GenDb::NewCf(g_viz_constants.FLOW_TABLE_PROT_DP, @@ -334,15 +334,15 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned64Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::LexicalUUIDType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::Unsigned16Type) (GenDb::DbDataType::Unsigned16Type) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* (VROUTER) index table */ (GenDb::NewCf(g_viz_constants.FLOW_TABLE_VROUTER, @@ -351,7 +351,7 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::Unsigned8Type), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), boost::assign::list_of @@ -359,15 +359,15 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned64Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::LexicalUUIDType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned8Type) (GenDb::DbDataType::Unsigned16Type) (GenDb::DbDataType::Unsigned16Type) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) ; @@ -377,9 +377,9 @@ void init_vizd_tables() { GenDb::DbDataTypeVec stat_row_onetag = boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned8Type) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) ; /* For Stat Tables that have two tags @@ -388,10 +388,10 @@ void init_vizd_tables() { GenDb::DbDataTypeVec stat_row_twotag = boost::assign::list_of (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::Unsigned8Type) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) ; vizd_stat_tables = @@ -404,12 +404,12 @@ void init_vizd_tables() { (GenDb::NewCf(g_viz_constants.STATS_TABLE_BY_STR_STR_TAG, stat_row_twotag, boost::assign::list_of - (GenDb::DbDataType::AsciiType) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* Stats Table by String,U64 tag * The schema is as follows: @@ -419,12 +419,12 @@ void init_vizd_tables() { (GenDb::NewCf(g_viz_constants.STATS_TABLE_BY_STR_U64_TAG, stat_row_twotag, boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned64Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* Stats Table by U64,Str tag * The schema is as follows: @@ -435,11 +435,11 @@ void init_vizd_tables() { stat_row_twotag, boost::assign::list_of (GenDb::DbDataType::Unsigned64Type) - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* Stats Table by U64,U64 tag * The schema is as follows: @@ -454,7 +454,7 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* Stats Table by U64 tag * The schema is as follows: @@ -468,7 +468,7 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* Stats Table by Double tag * The schema is as follows: @@ -482,7 +482,7 @@ void init_vizd_tables() { (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) /* Stats Table by String tag * The schema is as follows: @@ -492,11 +492,11 @@ void init_vizd_tables() { (GenDb::NewCf(g_viz_constants.STATS_TABLE_BY_STR_TAG, stat_row_onetag, boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) (GenDb::DbDataType::Unsigned32Type) (GenDb::DbDataType::LexicalUUIDType), boost::assign::list_of - (GenDb::DbDataType::AsciiType) + (GenDb::DbDataType::UTF8Type) )) ; @@ -505,11 +505,11 @@ void init_vizd_tables() { flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DIRECTION_ING]] = FlowTypeInfo(FlowRecordFields::FLOWREC_DIRECTION_ING, GenDb::DbDataType::Unsigned8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_SOURCEVN]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_SOURCEVN, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_SOURCEVN, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_SOURCEIP]] = FlowTypeInfo(FlowRecordFields::FLOWREC_SOURCEIP, GenDb::DbDataType::Unsigned32Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DESTVN]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_DESTVN, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_DESTVN, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DESTIP]] = FlowTypeInfo(FlowRecordFields::FLOWREC_DESTIP, GenDb::DbDataType::Unsigned32Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_PROTOCOL]] = @@ -523,15 +523,15 @@ void init_vizd_tables() { flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_TCP_FLAGS]] = FlowTypeInfo(FlowRecordFields::FLOWREC_TCP_FLAGS, GenDb::DbDataType::Unsigned8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_VM]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_VM, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_VM, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_INPUT_INTERFACE]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_INPUT_INTERFACE, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_INPUT_INTERFACE, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_OUTPUT_INTERFACE]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_OUTPUT_INTERFACE, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_OUTPUT_INTERFACE, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_MPLS_LABEL]] = FlowTypeInfo(FlowRecordFields::FLOWREC_MPLS_LABEL, GenDb::DbDataType::Unsigned32Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_REVERSE_UUID]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_REVERSE_UUID, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_REVERSE_UUID, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_SETUP_TIME]] = FlowTypeInfo(FlowRecordFields::FLOWREC_SETUP_TIME, GenDb::DbDataType::Unsigned64Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_TEARDOWN_TIME]] = @@ -553,17 +553,17 @@ void init_vizd_tables() { flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DIFF_PACKETS]] = FlowTypeInfo(FlowRecordFields::FLOWREC_DIFF_PACKETS, GenDb::DbDataType::Unsigned64Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_DATA_SAMPLE]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_DATA_SAMPLE, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_DATA_SAMPLE, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_ACTION]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_ACTION, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_ACTION, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_SG_RULE_UUID]] = FlowTypeInfo(FlowRecordFields::FLOWREC_SG_RULE_UUID, GenDb::DbDataType::LexicalUUIDType); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_NW_ACE_UUID]] = FlowTypeInfo(FlowRecordFields::FLOWREC_NW_ACE_UUID, GenDb::DbDataType::LexicalUUIDType); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_VROUTER_IP]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_VROUTER_IP, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_VROUTER_IP, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_OTHER_VROUTER_IP]] = - FlowTypeInfo(FlowRecordFields::FLOWREC_OTHER_VROUTER_IP, GenDb::DbDataType::AsciiType); + FlowTypeInfo(FlowRecordFields::FLOWREC_OTHER_VROUTER_IP, GenDb::DbDataType::UTF8Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_UNDERLAY_PROTO]] = FlowTypeInfo(FlowRecordFields::FLOWREC_UNDERLAY_PROTO, GenDb::DbDataType::Unsigned16Type); flow_msg2type_map[g_viz_constants.FlowRecordNames[FlowRecordFields::FLOWREC_UNDERLAY_SPORT]] = diff --git a/src/gendb/cdb_if.cc b/src/gendb/cdb_if.cc index b21cfd1e588..4a4e448f035 100644 --- a/src/gendb/cdb_if.cc +++ b/src/gendb/cdb_if.cc @@ -543,7 +543,13 @@ CdbIf::CdbIfTypeMapDef CdbIf::CdbIfTypeMap = DbEncodeDoubleComposite, DbDecodeDoubleComposite, DbEncodeDoubleNonComposite, - DbDecodeDoubleNonComposite)); + DbDecodeDoubleNonComposite)) + (GenDb::DbDataType::UTF8Type, + CdbIf::CdbIfTypeInfo("UTF8Type", + DbEncodeStringComposite, + DbDecodeStringComposite, + DbEncodeStringNonComposite, + DbDecodeStringNonComposite)); class CdbIf::CleanupTask : public Task { public: @@ -904,7 +910,7 @@ bool CdbIf::DbDataTypeVecToCompositeType(std::string& res, res = it->second.cassandra_type_; return true; } else { - res = "CompositeType("; + res = "org.apache.cassandra.db.marshal.CompositeType("; std::vector::const_iterator it = db_type.begin(); CdbIfTypeMapDef::iterator jt; @@ -914,7 +920,7 @@ bool CdbIf::DbDataTypeVecToCompositeType(std::string& res, it++; for (; it != db_type.end(); it++) { - res.append(", "); + res.append(","); if ((jt = CdbIfTypeMap.find(*it)) == CdbIfTypeMap.end()) return false; res.append(jt->second.cassandra_type_); @@ -1128,7 +1134,8 @@ bool CdbIf::Db_AddColumnfamily(const GenDb::NewCf& cf) { CdbIfCfInfo *cfinfo; if (Db_GetColumnfamily(&cfinfo, cf.cfname_)) { - if ((*cfinfo->cfdef_.get()).gc_grace_seconds != 0) { + if ((*cfinfo->cfdef_.get()).gc_grace_seconds != 0 || + DB_IsCfSchemaChanged(cfinfo->cfdef_.get(), &cf_def)) { CDBIF_LOG(DEBUG, "CFName: " << cf.cfname_ << " ID: " << (*cfinfo->cfdef_.get()).id); cf_def.__set_id((*cfinfo->cfdef_.get()).id); @@ -1190,7 +1197,8 @@ bool CdbIf::Db_AddColumnfamily(const GenDb::NewCf& cf) { CdbIfCfInfo *cfinfo; if (Db_GetColumnfamily(&cfinfo, cf.cfname_)) { - if ((*cfinfo->cfdef_.get()).gc_grace_seconds != 0) { + if (((*cfinfo->cfdef_.get()).gc_grace_seconds != 0) || + DB_IsCfSchemaChanged(cfinfo->cfdef_.get(), &cf_def)) { CDBIF_LOG(DEBUG, "CFName: " << cf.cfname_ << " ID: " << (*cfinfo->cfdef_.get()).id); cf_def.__set_id((*cfinfo->cfdef_.get()).id); @@ -1223,6 +1231,47 @@ bool CdbIf::Db_AddColumnfamily(const GenDb::NewCf& cf) { return true; } +bool CdbIf::DB_IsCfSchemaChanged(org::apache::cassandra::CfDef *cfdef, + org::apache::cassandra::CfDef *newcfdef) { + if (cfdef->key_validation_class != newcfdef->key_validation_class) { + return true; + } + + if ((newcfdef->default_validation_class.size()) && + (cfdef->default_validation_class != newcfdef->default_validation_class)) { + return true; + } + + if (cfdef->comparator_type != newcfdef->comparator_type) { + return true; + } + + if (cfdef->column_metadata.size() != newcfdef->column_metadata.size()) { + return true; + } + + std::vector::iterator cfdef_it = cfdef->column_metadata.begin(); + std::vector::iterator newcfdef_it; + + bool schema_changed; + for(; cfdef_it != cfdef->column_metadata.end(); cfdef_it++) { + schema_changed = true; + for(newcfdef_it = newcfdef->column_metadata.begin(); + newcfdef_it != newcfdef->column_metadata.end(); newcfdef_it++) { + if(cfdef_it->name == newcfdef_it->name && + cfdef_it->validation_class == newcfdef_it->validation_class) { + schema_changed = false; + break; + } + } + if(schema_changed) { + return true; + } + } + + return false; +} + bool CdbIf::Db_AsyncAddColumn(CdbIfColList &cl) { GenDb::ColList *new_colp(cl.gendb_cl); if (new_colp == NULL) { diff --git a/src/gendb/cdb_if.h b/src/gendb/cdb_if.h index da0461abd5d..5338542e5cf 100644 --- a/src/gendb/cdb_if.h +++ b/src/gendb/cdb_if.h @@ -89,7 +89,7 @@ class CdbIf : public GenDb::GenDbIf { DbDecodeCompositeFunc decode_composite_fn, DbEncodeNonCompositeFunc encode_non_composite_fn, DbDecodeNonCompositeFunc decode_non_composite_fn) : - cassandra_type_(cassandra_type), + cassandra_type_("org.apache.cassandra.db.marshal." + cassandra_type), encode_composite_fn_(encode_composite_fn), decode_composite_fn_(decode_composite_fn), encode_non_composite_fn_(encode_non_composite_fn), @@ -162,6 +162,8 @@ class CdbIf : public GenDb::GenDbIf { bool Db_AsyncAddColumn(CdbIfColList &cl); bool Db_AsyncAddColumnLocked(CdbIfColList &cl); void Db_BatchAddColumn(bool done); + bool DB_IsCfSchemaChanged(org::apache::cassandra::CfDef *cfdef, + org::apache::cassandra::CfDef *newcfdef); // Read static const int kMaxQueryRows = 5000; // API to get range of column data for a range of rows diff --git a/src/gendb/gendb.sandesh b/src/gendb/gendb.sandesh index cfc8c8bb199..ba57cdd4e5a 100644 --- a/src/gendb/gendb.sandesh +++ b/src/gendb/gendb.sandesh @@ -16,6 +16,7 @@ enum DbDataType { Unsigned64Type = 7, // uint64_t DoubleType = 8, // double + UTF8Type = 9, //utf-8 string } struct DbTableInfo { diff --git a/src/query_engine/select.cc b/src/query_engine/select.cc index 66b3232eebe..01930d134d8 100644 --- a/src/query_engine/select.cc +++ b/src/query_engine/select.cc @@ -464,6 +464,7 @@ query_status_t SelectQuery::process_query() { break; } case GenDb::DbDataType::AsciiType: + case GenDb::DbDataType::UTF8Type: { try { elem_value = boost::get(kt->second);