Closes-Bug: #1542596, Need storage support for sm lite

Change-Id: Idf4643069f263502c5a9e79221125baefbf93c89

src/client/preconfig.py

@@ -9,6 +9,7 @@
 import re
 import subprocess
 import sys
+from netaddr import IPNetwork
 
 # Testbed Converter Version
 __version__ = '1.0'
@@ -130,9 +131,16 @@ def set_mgmt_interface(self):
 
     def set_mgmt_ip_address(self, ):
         self.set_mgmt_interface()
+        self.sm = False
         for iface_dict in self.network['interfaces']:
-            if iface_dict['name'] == self.mgmt_iface:
-                self.ip, self.cidr = iface_dict['ip_address'].split('/')
+            if iface_dict['ip_address']:
+               ip = str(IPNetwork(iface_dict['ip_address']).ip)
+               cidr = str(IPNetwork(iface_dict['ip_address']).prefixlen)
+               if iface_dict['name'] == self.mgmt_iface:
+                   self.ip, self.cidr = ip, cidr
+               if ip == self.server_manager_ip:
+                   self.sm = True
+
 
     def connect(self):
         self.set_mgmt_ip_address()
@@ -422,7 +430,7 @@ def update_default_puppet(self):
 
     def remove_puppet_ssl(self):
         log.info('Remove puppet ssl for non-server-manager node')
-        if self.ip != self.server_manager_ip:
+        if not self.sm:
             if self.domain:
                 fqdn = '%s.%s' % (self.id, self.domain)
             else:

src/client/provision.sh

@@ -22,6 +22,8 @@ start_time=$(date +"%s")
 TESTBED="testbed.py"
 DEFAULT_DOMAIN=""
 CONTRAIL_PKG=""
+CONTRAIL_STORAGE_PKG=""
+STORAGE_KEYS_INI=""
 CLUSTER_ID="cluster_auto_$RANDOM"
 NO_SM_MON=""
 NO_SM_WEBUI=""
@@ -35,6 +37,8 @@ function usage()
     echo -e "\t-h --help"
     echo -e "\t-c|--contrail-package <pkg>"
     echo -e "\t-t|--testbed <testbed.py>"
+    echo -e "\t-cs|--contrail-storage-package <pkg>"
+    echo -e "\t-sk|--storage-keys-ini-file <file>"
     echo -e "\t-cid|--cluster-id <cluster-id>"
     echo ""
 }
@@ -60,6 +64,14 @@ while [[ $# > 0 ]]; do
         CLUSTER_ID="$2"
         shift # past argument
         ;;
+        -cs|--contrail-storage-package)
+        CONTRAIL_STORAGE_PKG="$2"
+        shift # past argument
+        ;;
+        -sk|--storage-keys-ini-file)
+        STORAGE_KEYS_INI="$2"
+        shift # past argument
+        ;;
         -h|--help)
         usage
         exit
@@ -96,8 +108,15 @@ TESTBED=$(get_real_path $TESTBED)
 echo "$space$arrow Convert testbed.py to server manager entities"
 # Convert testbed.py to server manager object json files
 optional_args=""
+if [ ! -z "$CONTRAIL_STORAGE_PKG" ]; then
+    optional_args="--contrail-storage-packages ${CONTRAIL_STORAGE_PKG}"
+fi
+if [ ! -z "$STORAGE_KEYS_INI" ]; then
+    STORAGE_KEYS_INI=$(python -c "import os; import sys; print(os.path.abspath(sys.argv[1]))" $STORAGE_KEYS_INI)
+    optional_args="--storage-keys-ini-file $STORAGE_KEYS_INI"
+fi
 if [ ! -z "$CLUSTER_ID" ]; then
-    optional_args="--cluster-id $CLUSTER_ID"
+    optional_args="$optional_args --cluster-id $CLUSTER_ID"
 fi
 cd $PROVISION_DIR && $SCRIPT_PATH/testbed_parser.py --testbed ${TESTBED} --contrail-packages ${CONTRAIL_PKG} $optional_args
 

src/client/storage_keys.ini

@@ -0,0 +1,4 @@
+[STORAGE-KEYS]
+storage_mon_secret = AQBM78tTEMz+GhAA3WiOXQI7UVdIy0YFFuTGdw==
+osd_bootstrap_key = AQCq7NFTeJUoBhAAlTVpxwWQJtBej/JDNhT6+Q==
+admin_key = AQDIgtNTgPLWARAAK6gs/fj8m88LnY9DwxJdYA==

src/client/testbed_parser.py

@@ -12,6 +12,7 @@
 import shutil
 import socket
 import sys
+import ConfigParser
 from tempfile import mkdtemp
 
 # Testbed Converter Version
@@ -72,6 +73,9 @@ def parse_args(args):
                             default=[],
                             help='Absolute path to Contrail Storage Package file, '\
                                  'Multiple files can be separated with space')
+        parser.add_argument('--storage-keys-ini-file',
+                            default=None,
+                            help='Provide storage keys for storage cluster creation')
         parser.add_argument('--cluster-id',
                             action='store',
                             default=None,
@@ -91,20 +95,37 @@ def parse_args(args):
             cliargs.contrail_storage_packages = [('contrail_storage_packages', pkg_file) \
                 for pkg_file in Utils.get_abspath(*Utils.is_file_exists(*cliargs.contrail_storage_packages))]
 
+        if cliargs.storage_keys_ini_file:
+            cliargs.storage_keys_ini_file = Utils.get_abspath(*Utils.is_file_exists(cliargs.storage_keys_ini_file))[0]
+
         # update log level and log file
         log_level = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
         cliargs.v = cliargs.v if cliargs.v <= 3 else 3
         Utils.initialize_logger(log_file=cliargs.log_file, log_level=log_level[cliargs.v])
         return cliargs
 
+    @staticmethod
+    def get_section_from_ini_file(file_name, section):
+        section_items = {}
+        if not file_name:
+            return section_items
+        ini_config = ConfigParser.SafeConfigParser()
+        ini_config.read(file_name)
+        if section in ini_config.sections():
+            section_items = dict(ini_config.items(section))
+        return section_items
+
     @staticmethod
     def converter(args):
         testsetup = TestSetup(testbed=args.testbed, cluster_id=args.cluster_id)
         testsetup.connect()
         testsetup.update()
-        server_json = ServerJsonGenerator(testsetup=testsetup)
+        server_json = ServerJsonGenerator(testsetup=testsetup, 
+                                          storage_packages=args.contrail_storage_packages)
         server_json.generate_json_file()
-        cluster_json = ClusterJsonGenerator(testsetup=testsetup)
+        storage_keys = Utils.get_section_from_ini_file(args.storage_keys_ini_file, 'STORAGE-KEYS')
+        cluster_json = ClusterJsonGenerator(testsetup=testsetup, 
+                                            storage_keys=storage_keys)
         cluster_json.generate_json_file()
         package_files = args.contrail_packages + args.contrail_storage_packages
         image_json = ImageJsonGenerator(testsetup=testsetup,
@@ -571,6 +592,7 @@ def __init__(self, **kwargs):
         name = kwargs.get('name', 'contrail')
         abspath = kwargs.get('abspath', None)
         self.package_files = kwargs.get('package_files', None)
+        self.storage_package_files = kwargs.get('storage_packages', None)
         self.jsonfile = abspath or '%s.json' % name
         self.cluster_id = self.testsetup.cluster_id or "cluster"
         self.dict_data = {}
@@ -696,10 +718,19 @@ def update_parameters_info(self, server_dict, hostobj):
                 server_dict['parameters']['disks'] = hostobj.storage_node_config['disks']
             else:
                 log.warn("No disks defined in storage_node_config")
+            if 'chassis' in hostobj.storage_node_config.keys():
+                server_dict['parameters']['storage_chassis_id'] = hostobj.storage_node_config['chassis']
+        if getattr(hostobj, 'roles', None) and self.storage_package_files:
+            if 'storage-master' in hostobj.roles or \
+               'storage-compute' in hostobj.roles:
+               package_type, package_file = self.storage_package_files[0]
+               version = ImageUtils.get_version(package_file, self.testsetup.os_type)
+               image_id = ImageUtils.get_image_id(version, package_type)
+               server_dict['parameters']['storage_repo_id'] = image_id
         return server_dict
 
+
     def update_dpdk_info(self, server_dict, hostobj):
-        server_dict['parameters'] = {}
         if hostobj.dpdk_config and \
             hostobj.dpdk_config.get('huge_pages', ''):
             server_dict['parameters']['huge_pages'] = hostobj.dpdk_config['huge_pages']
@@ -725,6 +756,7 @@ class ClusterJsonGenerator(BaseJsonGenerator):
     def __init__(self, testsetup, **kwargs):
         kwargs.update([('name', kwargs.get('name', 'cluster'))])
         super(ClusterJsonGenerator, self).__init__(testsetup=testsetup, **kwargs)
+        self.storage_keys = kwargs.get('storage_keys', None)
         self.dict_data = {"cluster": []}
 
     def _initialize(self):
@@ -745,6 +777,17 @@ def _initialize(self):
         self.set_if_defined('ext_routers', cluster_dict['parameters'],
                             destination_variable_name='external_bgp')
 
+        # Update storage keys
+        self.set_if_defined('storage_mon_secret', cluster_dict['parameters'],
+                            source_variable=self.storage_keys,
+                            function=dict.get)
+        self.set_if_defined('osd_bootstrap_key', cluster_dict['parameters'],
+                            source_variable=self.storage_keys,
+                            function=dict.get)
+        self.set_if_defined('admin_key', cluster_dict['parameters'],
+                            source_variable=self.storage_keys,
+                            function=dict.get)
+
         # Update ha details
         if getattr(self.testsetup, 'ha', None) is not None:
             self.set_if_defined('internal_vip',cluster_dict['parameters'],
@@ -807,25 +850,16 @@ def generate_json_file(self):
         self.dict_data['cluster'].append(cluster_dict)
         self.generate()
 
-class ImageJsonGenerator(BaseJsonGenerator):
-    def __init__(self, testsetup, package_files, **kwargs):
-        kwargs.update([('name', kwargs.get('name', 'image'))])
-        super(ImageJsonGenerator, self).__init__(testsetup=testsetup,
-                                                 package_files=package_files,
-                                                 **kwargs)
-        self.dict_data = {"image": []}
-        self.package_types = {'deb': 'package', 'rpm': 'package',
-                             'iso': 'image', 'tgz': 'package',
-                             'tar.gz': 'tgz'}
-
-    def get_version(self, package_file):
+class ImageUtils(object):
+    @staticmethod
+    def get_version(package_file, os_type):
         version = ''
         # only rpm or deb version can be retrieved
         if not (package_file.endswith('.rpm') or package_file.endswith('.deb') or package_file.endswith('.tgz')):
             return ""
-        if self.testsetup.os_type[0] in ['centos', 'fedora', 'redhat', 'centoslinux']:
+        if os_type[0] in ['centos', 'fedora', 'redhat', 'centoslinux']:
             cmd = "rpm -qp --queryformat '%%{VERSION}-%%{RELEASE}\\n' %s" % package_file
-        elif self.testsetup.os_type[0] in ['ubuntu']:
+        elif os_type[0] in ['ubuntu']:
             package_name = os.path.basename(package_file)
             if package_name.endswith('.tgz'):
                 exp = re.compile("[0-9].*")
@@ -838,9 +872,30 @@ def get_version(self, package_file):
                 version = pid.read().strip()
                 pid.flush()
         else:
-            raise Exception("ERROR: UnSupported OS Type (%s)" % self.testsetup.os_type)
+            raise Exception("ERROR: UnSupported OS Type (%s)" % os_type)
         return version
 
+    @staticmethod
+    def get_image_id(image_id, package_type):
+        replacables = ['.', '-', '~']
+        for r_item, item  in zip(['_'] * len(replacables), replacables):
+            image_id = image_id.replace(item, r_item)
+        image_id = package_type + '_' + image_id
+        return "image_%s" % image_id
+
+
+
+class ImageJsonGenerator(BaseJsonGenerator):
+    def __init__(self, testsetup, package_files, **kwargs):
+        kwargs.update([('name', kwargs.get('name', 'image'))])
+        super(ImageJsonGenerator, self).__init__(testsetup=testsetup,
+                                                 package_files=package_files,
+                                                 **kwargs)
+        self.dict_data = {"image": []}
+        self.package_types = {'deb': 'package', 'rpm': 'package',
+                             'iso': 'image', 'tgz': 'package',
+                             'tar.gz': 'tgz'}
+
     def get_category(self, package_file):
         category = 'package'
         ext = filter(package_file.endswith, self.package_types.keys())
@@ -865,14 +920,10 @@ def get_md5(self, package_file):
         return md5sum.split()[0]
 
     def _initialize(self, package_file, package_type):
-        image_id = version = self.get_version(package_file)
-        replacables = ['.', '-', '~']
-        for r_item, item  in zip(['_'] * len(replacables), replacables):
-            image_id = image_id.replace(item, r_item)
-        image_id = package_type + '_' + image_id
-
+        version = ImageUtils.get_version(package_file, self.testsetup.os_type)
+        image_id = ImageUtils.get_image_id(version, package_type)
         image_dict = {
-            "id": "image_%s" % image_id,
+            "id": image_id,
             "category": self.get_category(package_file),
             "version": version,
             "type": self.get_package_type(package_file, package_type),

src/sm_provision.sh

@@ -24,6 +24,9 @@ SOURCES_LIST="sources_list"
 TESTBED="testbed.py"
 DEFAULT_DOMAIN=""
 CONTRAIL_PKG=""
+CONTRAIL_STORAGE_PKG=""
+STORAGE_KEYS_INI=""
+HOSTIP=""
 INSTALL_SM_LITE="install_sm_lite"
 CLEANUP_PUPPET_AGENT=""
 NO_LOCAL_REPO=1
@@ -40,6 +43,8 @@ function usage()
     echo "$0"
     echo -e "\t-h --help"
     echo -e "\t-c|--contrail-package <pkg>"
+    echo -e "\t-cs|--contrail-storage-package <pkg>"
+    echo -e "\t-sk|--storage-keys-ini-file <file>"
     echo -e "\t-t|--testbed <testbed.py>"
     echo -e "\t-d|--default-domain <domain name>"
     echo -e  "\t-ni|--no-install-sm-lite"
@@ -48,6 +53,7 @@ function usage()
     echo -e "\t-nm|--no-sm-mon"
     echo -e "\t-nw|--no-sm-webui"
     echo -e "\t-swp|--sm-webui-port"
+    echo -e "\t-ip|--hostip"
     echo -e "\t-cid|--cluster-id <cluster-id>"
     echo ""
 }
@@ -66,6 +72,14 @@ while [[ $# > 0 ]]
         CONTRAIL_PKG="$2"
         shift # past argument
         ;;
+        -cs|--contrail-storage-package)
+        CONTRAIL_STORAGE_PKG="$2"
+        shift # past argument
+        ;;
+        -sk|--storage-keys-ini-file)
+        STORAGE_KEYS_INI="$2"
+        shift # past argument
+        ;;
         -t|--testbed)
         TESTBED="$2"
         shift # past argument
@@ -96,6 +110,10 @@ while [[ $# > 0 ]]
         CLUSTER_ID="$2"
         shift # past argument
         ;;
+        -ip|--hostip)
+        HOSTIP="$2"
+        shift # past argument
+        ;;
         -h|--help)
         usage
         exit
@@ -129,6 +147,10 @@ function get_real_path ()
 CONTRAIL_PKG=$(get_real_path $CONTRAIL_PKG)
 TESTBED=$(get_real_path $TESTBED)
 
+if [ -f "$CONTRAIL_STORAGE_PKG" ]; then
+    CONTRAIL_STORAGE_PKG=$(get_real_path $CONTRAIL_STORAGE_PKG)
+fi
+
 function unmount_contrail_local_repo()
 {
     echo "$arrow Removing contrail local repo - $LOCAL_REPO_DIR"
@@ -215,7 +237,11 @@ if [ "$INSTALL_SM_LITE" != "" ]; then
 
    echo "$arrow Install server manager without cobbler option"
    pushd /opt/contrail/contrail_server_manager >> $log_file 2>&1
-   ./setup.sh --all --smlite ${NO_SM_MON} ${NO_SM_WEBUI}
+   optional_args=""
+   if [ ! -z "$HOSTIP" ]; then
+       optional_args="--hostip=$HOSTIP"
+   fi
+   ./setup.sh --all --smlite ${NO_SM_MON} ${NO_SM_WEBUI} $optional_args
    popd >> $log_file 2>&1
 fi 
 
@@ -228,8 +254,15 @@ fi
 echo "$space$arrow Convert testbed.py to server manager entities"
 # Convert testbed.py to server manager object json files
 optional_args=""
+if [ ! -z "$CONTRAIL_STORAGE_PKG" ]; then
+    optional_args="--contrail-storage-packages ${CONTRAIL_STORAGE_PKG}"
+fi
+if [ ! -z "$STORAGE_KEYS_INI" ]; then
+    STORAGE_KEYS_INI=$(python -c "import os; import sys; print(os.path.abspath(sys.argv[1]))" $STORAGE_KEYS_INI)
+    optional_args="$optional_args --storage-keys-ini-file $STORAGE_KEYS_INI"
+fi
 if [ ! -z "$CLUSTER_ID" ]; then
-    optional_args="--cluster-id $CLUSTER_ID"
+    optional_args="$optional_args --cluster-id $CLUSTER_ID"
 fi
 cd $PROVISION_DIR && /opt/contrail/server_manager/client/testbed_parser.py --testbed ${TESTBED} --contrail-packages ${CONTRAIL_PKG} $optional_args