DM: porting fixes from R2.20 to master

Closes-Bug: #1464448
Closes-Bug: #1464453
Closes-Bug: #1464456
Closes-Bug: #1465904
Closes-Bug: #1466005

Change-Id: Iad3077505507e798d70258d76b40380527410eeb

src/config/device-manager/device_manager/db.py

@@ -110,8 +110,8 @@ def update(self, obj=None):
         self.name = obj['fq_name'][-1]
         self.management_ip = obj.get('physical_router_management_ip')
         self.dataplane_ip = obj.get('physical_router_dataplane_ip')
-        self.vendor = obj.get('physical_router_vendor_name')
-        self.product = obj.get('physical_router_product_name')
+        self.vendor = obj.get('physical_router_vendor_name', '')
+        self.product = obj.get('physical_router_product_name', '')
         self.vnc_managed = obj.get('physical_router_vnc_managed')
         self.user_credentials = obj.get('physical_router_user_credentials')
         self.junos_service_ports = obj.get('physical_router_junos_service_ports')
@@ -133,7 +133,7 @@ def delete(cls, uuid):
             return
         obj = cls._dict[uuid]
         obj.config_manager.delete_bgp_config()
-        self.uve_send(True)
+        obj.uve_send(True)
         obj.update_single_ref('bgp_router', {})
         obj.update_multiple_refs('virtual_network', {})
         del cls._dict[uuid]
@@ -181,6 +181,7 @@ def push_config(self):
                 self.config_manager.add_bgp_peer(peer.params['address'],
                                                  params, external)
             self.config_manager.set_bgp_config(bgp_router.params)
+            self.config_manager.set_global_routing_options(bgp_router.params)
             bgp_router_ips = bgp_router.get_all_bgp_router_ips()
             if self.dataplane_ip is not None and self.is_valid_ip(self.dataplane_ip):
                 self.config_manager.add_dynamic_tunnels(self.dataplane_ip,
@@ -210,10 +211,10 @@ def push_config(self):
                 vn_dict[vn_id] = [li.name]
 
         #for now, assume service port ifls unit numbers are always starts with 0 and goes on
-        service_port_id = 0
+        service_port_id = 1
         for vn_id, interfaces in vn_dict.items():
             vn_obj = VirtualNetworkDM.get(vn_id)
-            if vn_obj is None:
+            if vn_obj is None or vn_obj.vxlan_vni is None:
                 continue
             export_set = None
             import_set = None
@@ -278,7 +279,7 @@ def push_config(self):
                                                          False,
                                                          interfaces,
                                                          None,
-                                                         vn_obj.instance_ip_map)
+                                                         vn_obj.instance_ip_map, vn_obj.vxlan_vni)
 
         self.config_manager.send_bgp_config()
         self.uve_send()
@@ -313,6 +314,48 @@ def uve_send(self, deleted=False):
 
 # end PhysicalRouterDM
 
+class GlobalVRouterConfigDM(DBBase):
+    _dict = {}
+    obj_type = 'global_vrouter_config'
+    global_vxlan_id_mode = None
+
+    def __init__(self, uuid, obj_dict=None):
+        self.uuid = uuid
+        self.update(obj_dict)
+    # end __init__
+
+    def update(self, obj=None):
+        if obj is None:
+            obj = self.read_obj(self.uuid)
+        new_global_vxlan_id_mode = obj.get('vxlan_network_identifier_mode')
+        if GlobalVRouterConfigDM.global_vxlan_id_mode != new_global_vxlan_id_mode:
+            GlobalVRouterConfigDM.global_vxlan_id_mode = new_global_vxlan_id_mode
+            self.update_physical_routers()
+    # end update
+
+    def update_physical_routers(self):
+        for vn in VirtualNetworkDM.values():
+            vn.set_vxlan_vni()
+
+        for pr in PhysicalRouterDM.values():
+            pr.set_config_state()
+
+    #end update_physical_routers
+
+    @classmethod
+    def is_global_vxlan_id_mode_auto(cls):
+        if cls.global_vxlan_id_mode is not None and cls.global_vxlan_id_mode == 'automatic':
+            return True
+        return False
+
+    @classmethod
+    def delete(cls, uuid):
+        if uuid not in cls._dict:
+            return
+        obj = cls._dict[uuid]
+    # end delete
+# end GlobalVRouterConfigDM
+
 class GlobalSystemConfigDM(DBBase):
     _dict = {}
     obj_type = 'global_system_config'
@@ -535,7 +578,6 @@ def __init__(self, uuid, obj_dict=None):
         self.uuid = uuid
         self.physical_routers = set()
         self.router_external = False
-        self.vxlan_configured = False
         self.vxlan_vni = None
         self.gateways = None
         self.instance_ip_map = {}
@@ -551,15 +593,7 @@ def update(self, obj=None):
             self.router_external = obj['router_external']
         except KeyError:
             self.router_external = False
-        try:
-            prop = obj['virtual_network_properties']
-            if prop['vxlan_network_identifier'] is not None:
-                self.vxlan_configured = True
-                self.vxlan_vni = prop['vxlan_network_identifier']
-        except KeyError:
-            self.vxlan_configured = False 
-            self.vxlan_vni = None
-
+        self.set_vxlan_vni(obj)
         self.routing_instances = set([ri['uuid'] for ri in
                                       obj.get('routing_instances', [])])
         self.virtual_machine_interfaces = set(
@@ -586,6 +620,21 @@ def get_vrf_name(self, vrf_type=None):
         return vrf_name[:127]
     #end
 
+    def set_vxlan_vni(self, obj=None):
+        self.vxlan_vni = None
+        if obj is None:
+            obj = self.read_obj(self.uuid)
+        if GlobalVRouterConfigDM.is_global_vxlan_id_mode_auto():
+            self.vxlan_vni = obj.get('virtual_network_network_id')
+        else:
+            try:
+                prop = obj['virtual_network_properties']
+                if prop['vxlan_network_identifier'] is not None:
+                    self.vxlan_vni = prop['vxlan_network_identifier']
+            except KeyError:
+                pass
+    #end set_vxlan_vni
+
     def update_instance_ip_map(self):
         self.instance_ip_map = {}
         for vmi_uuid in self.virtual_machine_interfaces:
@@ -678,4 +727,5 @@ def delete(cls, uuid):
     'floating_ip': FloatingIpDM,
     'instance_ip': InstanceIpDM,
     'global_system_config': GlobalSystemConfigDM,
+    'global_vrouter_config': GlobalVRouterConfigDM,
 }

src/config/device-manager/device_manager/device_manager.py

@@ -39,7 +39,7 @@
 from cfgm_common.vnc_db import DBBase
 from db import BgpRouterDM, PhysicalRouterDM, PhysicalInterfaceDM, \
     LogicalInterfaceDM, VirtualMachineInterfaceDM, VirtualNetworkDM, RoutingInstanceDM, \
-    GlobalSystemConfigDM, FloatingIpDM, InstanceIpDM
+    GlobalSystemConfigDM, GlobalVRouterConfigDM, FloatingIpDM, InstanceIpDM
 from cfgm_common.dependency_tracker import DependencyTracker
 from sandesh.dm_introspect import ttypes as sandesh
 
@@ -189,6 +189,14 @@ def __init__(self, args=None):
             for fq_name, uuid in global_system_config_list:
                 GlobalSystemConfigDM.locate(uuid)
 
+        ok, global_vrouter_config_list = self._cassandra._cassandra_global_vrouter_config_list()
+        if not ok:
+            self.config_log('global vrouter config list returned error: %s' %
+                            global_vrouter_config_list)
+        else:
+            for fq_name, uuid in global_vrouter_config_list:
+                GlobalVRouterConfigDM.locate(uuid)
+
         ok, vn_list = self._cassandra._cassandra_virtual_network_list()
         if not ok:
             self.config_log('virtual network list returned error: %s' %

src/config/device-manager/device_manager/physical_router_config.py

@@ -70,6 +70,7 @@ def send_netconf(self, new_config, default_operation="merge",
             return
         self.commit_stats['netconf_enabled'] = True
         self.commit_stats['netconf_enabled_status'] = ''
+        start_time = None
         try:
             with manager.connect(host=self.management_ip, port=22,
                                  username=self.user_creds['username'],
@@ -109,8 +110,9 @@ def send_netconf(self, new_config, default_operation="merge",
                 self._logger.error("Router %s: %s" % (self.management_ip,
                                                       e.message))
                 self.commit_stats['commit_status_message'] = 'failed to apply config, router response: ' + e.message
-                self.commit_stats['last_commit_time'] = datetime.datetime.fromtimestamp(end_time).strftime('%Y-%m-%d %H:%M:%S')
-                self.commit_stats['last_commit_duration'] = str(time.time() - start_time)
+                if start_time is not None:
+                    self.commit_stats['last_commit_time'] = datetime.datetime.fromtimestamp(start_time).strftime('%Y-%m-%d %H:%M:%S')
+                    self.commit_stats['last_commit_duration'] = str(time.time() - start_time)
     # end send_config
 
     def add_dynamic_tunnels(self, tunnel_source_ip, ip_fabric_nets, bgp_router_ips):
@@ -129,8 +131,25 @@ def add_dynamic_tunnels(self, tunnel_source_ip, ip_fabric_nets, bgp_router_ips):
             etree.SubElement(dest_network, "name").text = bgp_router_ip + '/32'
     #end add_dynamic_tunnels
 
+    '''
+     ri_name: routing instance name to be configured on mx
+     import/export targets: routing instance import, export targets
+     prefixes: for l3 public vrf static routes, bug#1395938 
+     gateways: for l2 evpn, bug#1395944 
+     router_external: this indicates the routing instance configured is for 
+                      the public network
+     interfaces: logical interfaces to be part of vrf
+     fip_map: contrail instance ip to floating-ip map, used for snat & floating ip support
+     private_vni: This is used only for configuring snat vrf.
+                  private vn routing instance has an irb interface with the unit number 
+                  same as network vni. Snat vrf firewall filter should be applied 
+                  to private network irb interface, hence needed irb interface unit number. 
+                  This allows the traffic flow from private vrf to snat to 
+                  public and vice-versa
+    '''
     def add_routing_instance(self, ri_name, import_targets, export_targets,
-                             prefixes=[], gateways=[], router_external=False, interfaces=[], vni=None, fip_map=None):
+                             prefixes=[], gateways=[], router_external=False, 
+                             interfaces=[], vni=None, fip_map=None, private_vni=None):
         self.routing_instances[ri_name] = {'import_targets': import_targets,
                                         'export_targets': export_targets,
                                         'prefixes': prefixes,
@@ -142,35 +161,38 @@ def add_routing_instance(self, ri_name, import_targets, export_targets,
 
         ri_config = self.ri_config or etree.Element("routing-instances")
         policy_config = self.policy_config or etree.Element("policy-options")
-        ri = etree.SubElement(ri_config, "instance", operation="replace")
+        ri = etree.SubElement(ri_config, "instance")
         etree.SubElement(ri, "name").text = ri_name
         ri_opt = None
-        if vni is None:
+        if router_external:
+            ri_opt = etree.SubElement(ri, "routing-options")
+            static_config = etree.SubElement(ri_opt, "static")
+            route_config = etree.SubElement(static_config, "route")
+            etree.SubElement(route_config, "name").text = "0.0.0.0/0"
+            etree.SubElement(route_config, "next-table").text = "inet.0"
+
+        #for both l2 and l3
+        etree.SubElement(ri, "vrf-import").text = ri_name + "-import"
+        etree.SubElement(ri, "vrf-export").text = ri_name + "-export"
+
+        if vni is None or router_external:
             etree.SubElement(ri, "instance-type").text = "vrf"
-            for interface in interfaces:
-                if_element = etree.SubElement(ri, "interface")
-                etree.SubElement(if_element, "name").text = interface
-            etree.SubElement(ri, "vrf-import").text = ri_name + "-import"
-            etree.SubElement(ri, "vrf-export").text = ri_name + "-export"
-            etree.SubElement(ri, "vrf-table-label")
+            etree.SubElement(ri, "vrf-table-label")  #only for l3
+            if fip_map is None:
+                for interface in interfaces:
+                    if_element = etree.SubElement(ri, "interface")
+                    etree.SubElement(if_element, "name").text = interface
 
-            if prefixes:
-                ri_opt = etree.SubElement(ri, "routing-options")
-                static_config = etree.SubElement(ri_opt, "static")
+            if prefixes and fip_map is None:
+                if ri_opt is None:
+                    ri_opt = etree.SubElement(ri, "routing-options")
+                    static_config = etree.SubElement(ri_opt, "static")
                 for prefix in prefixes:
                     route_config = etree.SubElement(static_config, "route")
                     etree.SubElement(route_config, "name").text = prefix
                     etree.SubElement(route_config, "discard")
                 auto_export = "<auto-export><family><inet><unicast/></inet></family></auto-export>"
                 ri_opt.append(etree.fromstring(auto_export))
-
-            if router_external:
-                if ri_opt is None:
-                    ri_opt = etree.SubElement(ri, "routing-options")
-                    static_config = etree.SubElement(ri_opt, "static")
-                route_config = etree.SubElement(static_config, "route")
-                etree.SubElement(route_config, "name").text = "0.0.0.0/0"
-                etree.SubElement(route_config, "next-table").text = "inet.0"
         else:
             etree.SubElement(ri, "instance-type").text = "virtual-switch"
 
@@ -181,6 +203,8 @@ def add_routing_instance(self, ri_name, import_targets, export_targets,
             route_config = etree.SubElement(static_config, "route")
             etree.SubElement(route_config, "name").text = "0.0.0.0/0"
             etree.SubElement(route_config, "next-hop").text = interfaces[0]
+            if_element = etree.SubElement(ri, "interface")
+            etree.SubElement(if_element, "name").text = interfaces[0]
             public_vrf_ips = {}
             for pip in fip_map.values():
                 if pip["vrf_name"] not in public_vrf_ips:
@@ -192,6 +216,8 @@ def add_routing_instance(self, ri_name, import_targets, export_targets,
                 etree.SubElement(ri_public, "name").text = public_vrf
                 ri_opt = etree.SubElement(ri_public, "routing-options")
                 static_config = etree.SubElement(ri_opt, "static")
+                if_element = etree.SubElement(ri_public, "interface")
+                etree.SubElement(if_element, "name").text = interfaces[1]
 
                 for fip in fips:
                     route_config = etree.SubElement(static_config, "route")
@@ -227,7 +253,7 @@ def add_routing_instance(self, ri_name, import_targets, export_targets,
         # add firewall config for public VRF
         forwarding_options_config = self.forwarding_options_config
         firewall_config = self.firewall_config
-        if router_external:
+        if router_external or fip_map is not None:
             forwarding_options_config = self.forwarding_options_config or etree.Element("forwarding-options")
             fo = etree.SubElement(forwarding_options_config, "family")
             inet  = etree.SubElement(fo, "inet")
@@ -236,12 +262,17 @@ def add_routing_instance(self, ri_name, import_targets, export_targets,
             etree.SubElement(f, "input").text = "redirect_to_" + ri_name[:46] + "_vrf"
 
             firewall_config = self.firewall_config or etree.Element("firewall")
-            f = etree.SubElement(firewall_config, "filter")
+            fc = etree.SubElement(firewall_config, "family")
+            inet  = etree.SubElement(fc, "inet")
+            f = etree.SubElement(inet, "filter")
             etree.SubElement(f, "name").text = "redirect_to_" + ri_name[:46] + "_vrf"
             term = etree.SubElement(f, "term")
             etree.SubElement(term, "name").text= "t1"
-            from_ = etree.SubElement(term, "from")
-            if prefixes:
+            if fip_map is not None:
+                from_ = etree.SubElement(term, "from")
+                etree.SubElement(from_, "destination-address").text = ';'.join(fip_map.keys())
+            elif prefixes:
+                from_ = etree.SubElement(term, "from")
                 etree.SubElement(from_, "destination-address").text = ';'.join(prefixes)
             then_ = etree.SubElement(term, "then")
             etree.SubElement(then_, "routing-instance").text = ri_name
@@ -250,23 +281,31 @@ def add_routing_instance(self, ri_name, import_targets, export_targets,
             then_ = etree.SubElement(term, "then")
             etree.SubElement(then_, "accept")
 
+            if fip_map is not None:
+                interfaces_config = self.interfaces_config or etree.Element("interfaces")
+                irb_intf = etree.SubElement(interfaces_config, "interface")
+                etree.SubElement(irb_intf, "name").text = "irb"
+                intf_unit = etree.SubElement(irb_intf, "unit")
+                etree.SubElement(intf_unit, "name").text = str(private_vni)
+                family = etree.SubElement(intf_unit, "family")
+                inet = etree.SubElement(family, "inet")
+                f = etree.SubElement(inet, "filter")
+                input = etree.SubElement(f, "input")
+                etree.SubElement(input, "filter-name").text = "redirect_to_" + ri_name[:46] + "_vrf"
+
         # add L2 EVPN and BD config
         bd_config = None
         interfaces_config = self.interfaces_config
         proto_config = self.proto_config
-        if vni is not None and self.is_family_configured(self.bgp_params, "e-vpn"):
+        if (router_external==False and vni is not None and 
+                self.is_family_configured(self.bgp_params, "e-vpn")):
             etree.SubElement(ri, "vtep-source-interface").text = "lo0.0"
-            rt_element = etree.SubElement(ri, "vrf-target")
-            #fix me, check if this is correct target value for vrf-target
-            for route_target in import_targets:
-                etree.SubElement(rt_element, "community").text = route_target
             bd_config = etree.SubElement(ri, "bridge-domains")
             bd= etree.SubElement(bd_config, "domain")
             etree.SubElement(bd, "name").text = "bd-" + str(vni)
-            etree.SubElement(bd, "vlan-id").text = str(vni)
+            etree.SubElement(bd, "vlan-id").text = 'none'
             vxlan = etree.SubElement(bd, "vxlan")
             etree.SubElement(vxlan, "vni").text = str(vni)
-            etree.SubElement(vxlan, "ingress-node-replication")
             for interface in interfaces:
                 if_element = etree.SubElement(bd, "interface")
                 etree.SubElement(if_element, "name").text = interface
@@ -392,6 +431,12 @@ def add_routing_instance(self, ri_name, import_targets, export_targets,
         self.ri_config = ri_config
     # end add_routing_instance
 
+    def set_global_routing_options(self, bgp_params):
+        if bgp_params['address'] is not None:
+            self.global_routing_options_config = etree.Element("routing-options")
+            etree.SubElement(self.global_routing_options_config, "router-id").text = bgp_params['address']
+    #end set_global_routing_options
+
     def is_family_configured(self, params, family_name):
         if params is None or params.get('address_families') is None:
             return False
@@ -467,6 +512,7 @@ def reset_bgp_config(self):
         self.policy_config = None
         self.firewall_config = None
         self.forwarding_options_config = None
+        self.global_routing_options_config = None
         self.proto_config = None
         self.route_targets = set()
         self.bgp_peers = {}
@@ -556,6 +602,8 @@ def send_bgp_config(self):
             config_list.append(self.firewall_config)
         if self.forwarding_options_config is not None:
             config_list.append(self.forwarding_options_config)
+        if self.global_routing_options_config is not None:
+            config_list.append(self.global_routing_options_config)
         if self.proto_config is not None:
             config_list.append(self.proto_config)
         self.send_netconf(config_list)