Merge "Api server should limit maximum number of concurrent requests,…

… when customers limit the fd's they should be allowed to limit the maximum number of concurrent requests to api-server." into R2.1

src/config/api-server/SConscript

@@ -68,7 +68,8 @@ local_sources = [
            'vnc_perms.py',
            'vnc_addr_mgmt.py',
            'vnc_quota.py',
-           'provision_defaults.py'
+           'provision_defaults.py',
+           'vnc_bottle.py',
           ]
 
 local_sources_rules = []

src/config/api-server/tests/test_crud_basic.py

@@ -998,6 +998,74 @@ def test_list_bulk_collection(self):
 
 # end class TestVncCfgApiServer
 
+class TestVncCfgApiServerRequests(test_case.ApiServerTestCase):
+    """ Tests to verify the max_requests config parameter of api-server."""
+    def __init__(self, *args, **kwargs):
+        super(TestVncCfgApiServerRequests, self).__init__(*args, **kwargs)
+        self._config_knobs.extend([('DEFAULTS', 'max_requests', 10),])
+
+
+    def api_requests(self, orig_vn_read, count):
+        api_server = test_common.vnc_cfg_api_server.server
+        self.blocked = True
+        def slow_response_on_vn_read(*args, **kwargs):
+            while self.blocked:
+                gevent.sleep(1)
+            return orig_vn_read(*args, **kwargs)
+
+        api_server._db_conn._cassandra_db._cassandra_virtual_network_read = slow_response_on_vn_read
+
+        logger.info("Creating a test VN object.")
+        test_obj = self._create_test_object()
+        logger.info("Making max_requests(%s) to api server" % (count - 1))
+        def vn_read():
+            self._vnc_lib.virtual_network_read(id=test_obj.uuid)
+            gevent.sleep(0)
+
+        for i in range(count):
+            gevent.spawn(vn_read)
+        gevent.sleep(1)
+
+    def test_within_max_api_requests(self):
+        api_server = test_common.vnc_cfg_api_server.server
+        orig_vn_read = api_server._db_conn._cassandra_db._cassandra_virtual_network_read
+        try:
+            self.api_requests(orig_vn_read, 5)
+            logger.info("Making one more requests well within the max_requests to api server")
+            vn_name = self.id() + 'testvn'
+            try:
+                greenlet = gevent.spawn(self.create_virtual_network, vn_name, '10.1.1.0/24')
+                gevent.sleep(0)
+                vn_obj = greenlet.get(timeout=3)
+            except gevent.timeout.Timeout as e:
+                self.assertFalse(greenlet.successful(), 'Request failed unexpectedly')
+            else:
+                self.assertEqual(vn_obj.name, vn_name)
+        finally:
+            api_server._db_conn._cassandra_db._cassandra_virtual_network_read = orig_vn_read
+
+    def test_err_on_max_api_requests(self):
+        api_server = test_common.vnc_cfg_api_server.server
+        orig_vn_read = api_server._db_conn._cassandra_db._cassandra_virtual_network_read
+        try:
+            self.api_requests(orig_vn_read, 11)
+            logger.info("Making one more requests (max_requests + 1) to api server")
+            try:
+                greenlet = gevent.spawn(self.create_virtual_network, 'testvn', '10.1.1.0/24')
+                gevent.sleep(0)
+                greenlet.get(timeout=3)
+            except gevent.timeout.Timeout as e:
+                logger.info("max_requests + 1 failed as expected.")
+                self.blocked = False
+                self.assertFalse(False, greenlet.successful())
+            else:
+                self.assertTrue(False, 'Request succeeded unexpectedly')
+        finally:
+            api_server._db_conn._cassandra_db._cassandra_virtual_network_read = orig_vn_read
+
+# end class TestVncCfgApiServerRequests
+
+
 class TestLocalAuth(test_case.ApiServerTestCase):
     def __init__(self, *args, **kwargs):
         super(TestLocalAuth, self).__init__(*args, **kwargs)

src/config/api-server/vnc_auth_keystone.py

@@ -21,6 +21,7 @@
     pass
 
 from pysandesh.gen_py.sandesh.ttypes import SandeshLevel
+from vnc_bottle import get_bottle_server
 
 # Open port for access to API server for trouble shooting
 
@@ -63,7 +64,8 @@ def local_auth_check(*args, **kwargs):
 
     def start_http_server(self):
         self._http_app.run(
-            host=self._http_host, port=self._http_port, server='gevent')
+            host=self._http_host, port=self._http_port,
+            server=get_bottle_server(self._conf_info.get('max_requests')))
     # end start_http_server
 # end class LocalAuth
 
@@ -137,6 +139,7 @@ def __init__(self, server_mgr, args):
             'admin_password': args.admin_password,
             'admin_tenant_name': args.admin_tenant_name,
             'admin_port': args.admin_port,
+            'max_requests': args.max_requests,
         }
         self._server_mgr = server_mgr
         self._auth_method = args.auth

src/config/api-server/vnc_bottle.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2015 Juniper Networks, Inc. All rights reserved.
+#
+
+def get_bottle_server(pool_size):
+    """
+    Custom gevent pool server for bottle
+    """
+    from bottle import GeventServer
+    from gevent.pool import Pool
+
+    class GeventPoolServer(GeventServer):
+        """ Gevent server with limited pool size
+        """
+        def __init__(self, host='127.0.0.1', port=8080, **options):
+            super(GeventPoolServer, self ).__init__(host, port,
+                spawn=Pool(size=pool_size), **options)
+
+    return GeventPoolServer

src/config/api-server/vnc_cfg_api_server.py

@@ -97,6 +97,7 @@
     NodeStatus
 
 from sandesh.traces.ttypes import RestApiTrace
+from vnc_bottle import get_bottle_server
 
 _WEB_HOST = '0.0.0.0'
 _WEB_PORT = 8082
@@ -875,6 +876,7 @@ def _parse_args(self, args_str):
             'rabbit_vhost': None,
             'rabbit_max_pending_updates': '4096',
             'cluster_id': '',
+            'max_requests': 1024,
         }
         # ssl options
         secopts = {
@@ -1042,6 +1044,9 @@ def _parse_args(self, args_str):
         parser.add_argument(
             "--cluster_id",
             help="Used for database keyspace separation")
+        parser.add_argument(
+            "--max_requests", type=int,
+            help="Maximum number of concurrent requests served by api server")
         self._args = parser.parse_args(remaining_argv)
         self._args.config_sections = config
         if type(self._args.cassandra_server_list) is str:
@@ -1718,7 +1723,7 @@ def main(args_str=None):
 
     try:
         bottle.run(app=pipe_start_app, host=server_ip, port=server_port,
-                   server='gevent')
+                   server=get_bottle_server(server._args.max_requests))
     except KeyboardInterrupt:
         # quietly handle Ctrl-C
         pass