Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
1. Add option for cloud admin access only for analytics REST API
Allow cloud admin role access only for analytics REST API controlled
via --cloud_admin_access_only currently defaulted to False but will default
to True once provisioning changes are done. contrail-analytics-api will
validate role from the X-Auth-Token header via vnc_api/contrail-api. For
debug/administration a localhost bound port 8181 - --admin_port is provided
that requires basic HTTP access authentication.

Clients of analytics REST API - contrail-flows, contrail-logs, contrail-stats,
contrail-topology are changed to use admin port. contrail-svc-monitor is changed
to use auth token.

Conflicts:
	src/opserver/SConscript

Partial-Bug: #1461175
(cherry picked from commit 5492f71)

2. Rename cloud_admin_access_only to multi_tenancy in contrail-analytics-api

Closes-Bug: #1461175
(cherry picked from commit 36df099)

3. for bool option, a conversion from string to bool is required.
Closes-Bug: #1595044

(cherry picked from commit 1d6b81b)

4. Change cloud admin role name to "cloud-admin" from "admin" for
analytics API access

Closes-Bug: #1600699
(cherry picked from commit 8c13101)

5. Rename multi_tenancy to aaa_mode for analytics API

Handle keystone v2 and v3 token infos returned by
VNC API. Enable cloud-admin-only aaa_mode by default

Change analytics DB and underlay to overlay mapper to
use local admin port when quering opserver

Do not cache auth_token in vnc lib

Closes-Bug: #1599654
(cherry picked from commit a2a7c92)

6. Changes to bring analytics authenticated access in sync with config

  1. Rename aaa_mode value cloud-admin-only to cloud-admin
  2. CLOUD_ADMIN_ROLE defaults to admin instead of cloud-admin

Partial-Bug: #1607563
(cherry picked from commit 42db6e3)

7. Fix missing import of OpServerUtils in analytics_db.py

Closes-Bug: #1609054
(cherry picked from commit cf5f056)

8. Remove aaa_mode value cloud-admin-only

Closes-Bug: #1609987

9. Keep on trying to create VNC API client from analytics API

The gevent that creates the VNC API client was exiting due to
authentication failure exception. Changed code to handle all
exceptions and keep on trying to create the API client. The
node status will show the API connection down in case we are
not able to create the VNC API client.

Closes-Bug: #1611158
(cherry picked from commit 8072aa5)

10. Keystone middleware doesn't like if token is unicode. It must be converted
to string before validation.

Fixes-Bug: #1604773
(cherry picked from commit 18df643)

11. Change the obj-perms API to pass in the user token in HTTP headers

With PKI tokens, when user token was passed in query parameters for
obj-perms API the token was getting truncated. Changed the API
to accept user token in X-USER-TOKEN HTTP header.

Closes-Bug: #1614376

12.
  1. Called once check moved from _list_collection to list_bulk_collection_http_post, due to refractoring bug.
  2. Removed the local API server teardown for class TestPermissions
  3. Project's within class TestPermissions appended with self.id(), to create unique Project for each testcase.
Closes-Bug: 1555323

(cherry picked from commit a8ac59a)

Change-Id: Ia6bb36b37a86b33d87f304e9c784fa6fd780222b
  • Loading branch information
Megh Bhatt committed Aug 23, 2016
1 parent 94725ab commit 761ffd9
Show file tree
Hide file tree
Showing 30 changed files with 605 additions and 204 deletions.
Expand Up @@ -3,6 +3,7 @@
#
import requests, json
from requests.exceptions import ConnectionError
from requests.auth import HTTPBasicAuth

class AnalyticApiClient(object):
def __init__(self, cfg):
Expand All @@ -23,7 +24,8 @@ def init_client(self):
def _get_url_json(self, url):
if url is None:
return {}
page = self.client.get(url)
page = self.client.get(url, auth=HTTPBasicAuth(
self.config.admin_user(), self.config.admin_password()))
if page.status_code == 200:
return json.loads(page.text)
raise ConnectionError, "bad request " + url
Expand Down
11 changes: 9 additions & 2 deletions src/analytics/contrail-topology/contrail_topology/config.py
Expand Up @@ -4,7 +4,8 @@
import argparse, os, ConfigParser, sys, re
from pysandesh.sandesh_base import *
from pysandesh.gen_py.sandesh.ttypes import SandeshLevel
from sandesh_common.vns.constants import ModuleNames, HttpPortTopology, API_SERVER_DISCOVERY_SERVICE_NAME
from sandesh_common.vns.constants import ModuleNames, HttpPortTopology, \
API_SERVER_DISCOVERY_SERVICE_NAME, OpServerAdminPort
from sandesh_common.vns.ttypes import Module
import discoveryclient.client as discovery_client
import traceback
Expand Down Expand Up @@ -69,7 +70,7 @@ def parse(self):

defaults = {
'collectors' : None,
'analytics_api' : ['127.0.0.1:8081'],
'analytics_api' : ['127.0.0.1:' + str(OpServerAdminPort)],
'log_local' : False,
'log_level' : SandeshLevel.SYS_DEBUG,
'log_category' : '',
Expand Down Expand Up @@ -224,6 +225,12 @@ def frequency(self):
def http_port(self):
return self._args.http_server_port

def admin_user(self):
return self._args.admin_user

def admin_password(self):
return self._args.admin_password

def sandesh_send_rate_limit(self):
return self._args.sandesh_send_rate_limit

Expand Down
24 changes: 12 additions & 12 deletions src/api-lib/vnc_api.py
Expand Up @@ -1044,8 +1044,6 @@ def virtual_network_subnet_ip_count(self, vnobj, subnet_list):
#end virtual_network_subnet_ip_count

def get_auth_token(self):
if self._auth_token:
return self._auth_token
self._headers = self._authenticate(headers=self._headers)
return self._auth_token

Expand Down Expand Up @@ -1149,19 +1147,21 @@ def set_user_roles(self, roles):
self._headers['X-API-ROLE'] = (',').join(roles)
#end set_user_roles

"""
validate user token. Optionally, check token authorization for an object.
rv {'token_info': <token-info>, 'permissions': 'RWX'}
"""
def obj_perms(self, token, obj_uuid=None):
"""
validate user token. Optionally, check token authorization for an object.
rv {'token_info': <token-info>, 'permissions': 'RWX'}
"""
query = 'token=%s' % token
if obj_uuid:
query += '&uuid=%s' % obj_uuid
self._headers['X-USER-TOKEN'] = token
query = 'uuid=%s' % obj_uuid if obj_uuid else ''
try:
rv = self._request_server(rest.OP_GET, "/obj-perms", data=query)
return json.loads(rv)
rv_json = self._request_server(rest.OP_GET, "/obj-perms", data=query)
rv = json.loads(rv_json)
except PermissionDenied:
return None
rv = None
finally:
del self._headers['X-USER-TOKEN']
return rv

# change object ownsership
def chown(self, obj_uuid, owner):
Expand Down
12 changes: 6 additions & 6 deletions src/config/api-server/tests/test_crud_basic.py
Expand Up @@ -1176,14 +1176,14 @@ def test_list_bulk_collection(self):
vmi_uuids = [o.uuid for o in vmi_objs]

logger.info("Querying VNs by obj_uuids.")
flexmock(self._api_server).should_call('_list_collection').once()
flexmock(self._api_server).should_call('list_bulk_collection_http_post').once()
ret_list = self._vnc_lib.resource_list('virtual-network',
obj_uuids=vn_uuids)
ret_uuids = [ret['uuid'] for ret in ret_list['virtual-networks']]
self.assertThat(set(vn_uuids), Equals(set(ret_uuids)))

logger.info("Querying RIs by parent_id.")
flexmock(self._api_server).should_call('_list_collection').once()
flexmock(self._api_server).should_call('list_bulk_collection_http_post').once()
ret_list = self._vnc_lib.resource_list('routing-instance',
parent_id=vn_uuids)
ret_uuids = [ret['uuid']
Expand All @@ -1192,15 +1192,15 @@ def test_list_bulk_collection(self):
Equals(set(ret_uuids) & set(ri_uuids)))

logger.info("Querying VMIs by back_ref_id.")
flexmock(self._api_server).should_call('_list_collection').once()
flexmock(self._api_server).should_call('list_bulk_collection_http_post').once()
ret_list = self._vnc_lib.resource_list('virtual-machine-interface',
back_ref_id=vn_uuids)
ret_uuids = [ret['uuid']
for ret in ret_list['virtual-machine-interfaces']]
self.assertThat(set(vmi_uuids), Equals(set(ret_uuids)))

logger.info("Querying VMIs by back_ref_id and extra fields.")
flexmock(self._api_server).should_call('_list_collection').once()
flexmock(self._api_server).should_call('list_bulk_collection_http_post').once()
ret_list = self._vnc_lib.resource_list('virtual-machine-interface',
back_ref_id=vn_uuids,
fields=['virtual_network_refs'])
Expand All @@ -1212,14 +1212,14 @@ def test_list_bulk_collection(self):
set(vn_uuids))

logger.info("Querying RIs by parent_id and filter.")
flexmock(self._api_server).should_call('_list_collection').once()
flexmock(self._api_server).should_call('list_bulk_collection_http_post').once()
ret_list = self._vnc_lib.resource_list('routing-instance',
parent_id=vn_uuids,
filters={'display_name':'%s-ri-5' %(self.id())})
self.assertThat(len(ret_list['routing-instances']), Equals(1))

logger.info("Querying VNs by obj_uuids for children+backref fields.")
flexmock(self._api_server).should_call('_list_collection').once()
flexmock(self._api_server).should_call('list_bulk_collection_http_post').once()
ret_objs = self._vnc_lib.resource_list('virtual-network',
detail=True, obj_uuids=vn_uuids, fields=['routing_instances',
'virtual_machine_interface_back_refs'])
Expand Down
4 changes: 1 addition & 3 deletions src/config/api-server/tests/test_perms2.py
Expand Up @@ -112,9 +112,7 @@ def api_acl_name(self):
return rg_name

def check_perms(self, obj_uuid):
query = 'token=%s&uuid=%s' % (self.vnc_lib.get_auth_token(), obj_uuid)
rv = self.vnc_lib._request_server(rest.OP_GET, "/obj-perms", data=query)
rv = json.loads(rv)
rv = self.vnc_lib.obj_perms(self.vnc_lib.get_auth_token(), obj_uuid)
return rv['permissions']

# display resource id-perms
Expand Down
4 changes: 2 additions & 2 deletions src/config/api-server/vnc_cfg_api_server.py
Expand Up @@ -1756,10 +1756,10 @@ def documentation_http_get(self, filename):
# end documentation_http_get

def obj_perms_http_get(self):
if 'token' not in get_request().query:
if 'HTTP_X_USER_TOKEN' not in get_request().environ:
raise cfgm_common.exceptions.HttpError(
400, 'User token needed for validation')
user_token = get_request().query.token.encode("ascii")
user_token = get_request().environ['HTTP_X_USER_TOKEN'].encode("ascii")

# get permissions in internal context
try:
Expand Down
9 changes: 6 additions & 3 deletions src/config/common/analytics_client.py
Expand Up @@ -34,12 +34,13 @@ def __init__(self, endpoint, data={}):
self.endpoint = endpoint
self.data = data

def request(self, path, fqdn_uuid, data=None):
def request(self, path, fqdn_uuid, user_token=None,
data=None):
req_data = dict(self.data)
if data:
req_data.update(data)

req_params = self._get_req_params(data=req_data)
req_params = self._get_req_params(user_token, data=req_data)

url = urlparse.urljoin(self.endpoint, path + fqdn_uuid)
resp = requests.get(url, **req_params)
Expand All @@ -51,13 +52,15 @@ def request(self, path, fqdn_uuid, data=None):

return resp.json()

def _get_req_params(self, data=None):
def _get_req_params(self, user_token, data=None):
req_params = {
'headers': {
'Accept': 'application/json'
},
'data': data,
'allow_redirects': False,
}
if user_token:
req_params['headers']['X-AUTH-TOKEN'] = user_token

return req_params
4 changes: 3 additions & 1 deletion src/config/common/tests/tools/install_venv_common.py
Expand Up @@ -120,7 +120,9 @@ def pip_install(self, find_links, *args):
find_links_str = ' '.join('--find-links file://'+x for x in find_links)
cmd_array = ['%stools/with_venv.sh' %(os.environ.get('tools_path', '')),
'python', '.venv/bin/pip', 'install',
'--upgrade', '--no-cache-dir']
'--upgrade']
if not args[0].startswith('pip'):
cmd_array.extend(['--no-cache-dir'])
for link in find_links:
cmd_array.extend(['--find-links', 'file://'+link])
self.run_command(cmd_array + list(args),
Expand Down
Expand Up @@ -106,7 +106,8 @@ def query_uve(self, filter_string):
path = "/analytics/uves/vrouter/"
response_dict = {}
try:
response = self._analytics.request(path, filter_string)
response = self._analytics.request(path, filter_string,
user_token=self._vnc_lib.get_auth_token())
for values in response['value']:
response_dict[values['name']] = values['value']
except analytics_client.OpenContrailAPIFailed:
Expand Down
5 changes: 4 additions & 1 deletion src/opserver/SConscript
Expand Up @@ -13,6 +13,7 @@ OpEnv = BuildEnv.Clone()
setup_sources = [
'setup.py',
'MANIFEST.in',
'requirements.txt',
]

setup_sources_rules = []
Expand Down Expand Up @@ -41,7 +42,9 @@ local_sources = [
'partition_handler.py',
'consistent_schdlr.py',
'gendb_move_tables.py',
'alarm_notify.py'
'alarm_notify.py',
'vnc_cfg_api_client.py',
'opserver_local.py',
]

plugins_sources = [
Expand Down
16 changes: 11 additions & 5 deletions src/opserver/analytics_db.py
Expand Up @@ -34,6 +34,7 @@
from cassandra.query import named_tuple_factory
from cassandra.query import PreparedStatement, tuple_factory
import platform
from opserver_util import OpServerUtils

class AnalyticsDb(object):
def __init__(self, logger, cassandra_server_list,
Expand Down Expand Up @@ -603,20 +604,25 @@ def db_purge_thrift(self, purge_cutoff, purge_id):
return (total_rows_deleted, purge_error_details)
# end db_purge

def get_dbusage_info(self, rest_api_ip, rest_api_port):
def get_dbusage_info(self, ip, port, user, password):
"""Collects database usage information from all db nodes
Returns:
A dictionary with db node name as key and db usage in % as value
"""

to_return = {}
try:
uve_url = "http://" + rest_api_ip + ":" + str(rest_api_port) + "/analytics/uves/database-nodes?cfilt=DatabaseUsageInfo"
node_dburls = json.loads(urllib2.urlopen(uve_url).read())
uve_url = "http://" + ip + ":" + str(port) + \
"/analytics/uves/database-nodes?cfilt=DatabaseUsageInfo"
data = OpServerUtils.get_url_http(uve_url, user, password)
node_dburls = json.loads(data)

for node_dburl in node_dburls:
# calculate disk usage percentage for analytics in each cassandra node
db_uve_state = json.loads(urllib2.urlopen(node_dburl['href']).read())
# calculate disk usage percentage for analytics in each
# cassandra node
db_uve_data = OpServerUtils.get_url_http(node_dburl['href'],
user, password)
db_uve_state = json.loads(db_uve_data)
db_usage_in_perc = (100*
float(db_uve_state['DatabaseUsageInfo']['database_usage'][0]['analytics_db_size_1k'])/
float(db_uve_state['DatabaseUsageInfo']['database_usage'][0]['disk_space_available_1k'] +
Expand Down
15 changes: 11 additions & 4 deletions src/opserver/flow.py
Expand Up @@ -76,7 +76,7 @@ def run(self):
def parse_args(self):
"""
Eg. python flow.py --analytics-api-ip 127.0.0.1
--analytics-api-port 8081
--analytics-api-port 8181
--vrouter a6s23
--source-vn default-domain:default-project:vn1
--destination-vn default-domain:default-project:vn2
Expand All @@ -94,7 +94,7 @@ def parse_args(self):
"""
defaults = {
'analytics_api_ip': '127.0.0.1',
'analytics_api_port': '8081',
'analytics_api_port': '8181',
'start_time': 'now-10m',
'end_time': 'now',
'direction' : 'ingress',
Expand Down Expand Up @@ -139,6 +139,11 @@ def parse_args(self):
help="Show vmi uuid information")
parser.add_argument(
"--verbose", action="store_true", help="Show internal information")
parser.add_argument(
"--admin-user", help="Name of admin user", default="admin")
parser.add_argument(
"--admin-password", help="Password of admin user",
default="contrail123")
self._args = parser.parse_args()

try:
Expand Down Expand Up @@ -332,13 +337,15 @@ def query(self):
json.dumps(flow_query.__dict__))
print ''
resp = OpServerUtils.post_url_http(
flow_url, json.dumps(flow_query.__dict__))
flow_url, json.dumps(flow_query.__dict__), self._args.admin_user,
self._args.admin_password)
result = {}
if resp is not None:
resp = json.loads(resp)
qid = resp['href'].rsplit('/', 1)[1]
result = OpServerUtils.get_query_result(
self._args.analytics_api_ip, self._args.analytics_api_port, qid)
self._args.analytics_api_ip, self._args.analytics_api_port, qid,
self._args.admin_user, self._args.admin_password)
return result
# end query

Expand Down
45 changes: 27 additions & 18 deletions src/opserver/introspect_util.py
Expand Up @@ -2,28 +2,37 @@
# Copyright (c) 2013 Juniper Networks, Inc. All rights reserved.
#

import urllib, urllib2
import urllib
import xmltodict
import json
import requests
from lxml import etree
import socket

from requests.auth import HTTPBasicAuth

class JsonDrv (object):

def _http_con(self, url):
return urllib2.urlopen(url)

def load(self, url):
return json.load(self._http_con(url))

def load(self, url, user, password):
try:
if user and password:
auth=HTTPBasicAuth(user, password)
else:
auth=None
resp = requests.get(url, auth=auth)
return json.loads(resp.text)
except requests.ConnectionError, e:
print "Socket Connection error : " + str(e)
return None

class XmlDrv (object):

def load(self, url):
def load(self, url, user, password):
try:
resp = requests.get(url)
if user and password:
auth=HTTPBasicAuth(user, password)
else:
auth=None
resp = requests.get(url, auth=auth)
return etree.fromstring(resp.text)
except requests.ConnectionError, e:
print "Socket Connection error : " + str(e)
Expand Down Expand Up @@ -54,14 +63,14 @@ def _mk_url_str(self, path, query):
return path+query_str
return "http://%s:%d/%s%s" % (self._ip, self._port, path, query_str)

def dict_get(self, path='', query=None, drv=None):
try:
if path:
if drv is not None:
return drv().load(self._mk_url_str(path, query))
return self._drv.load(self._mk_url_str(path, query))
except urllib2.HTTPError:
return None
def dict_get(self, path='', query=None, drv=None, user=None,
password=None):
if path:
if drv is not None:
return drv().load(self._mk_url_str(path, query), user,
password)
return self._drv.load(self._mk_url_str(path, query), user,
password)
# end dict_get


Expand Down

0 comments on commit 761ffd9

Please sign in to comment.