Dynamic resource allocation

backend/tomato/api/elements.py

@@ -298,6 +298,7 @@ def element_usage(id): #@ReservedAssignment
 	"""
 	el = _getElement(id)
 	return el.totalUsage.info()	
+
 
 from .. import elements, currentUser
 from topology import _getTopology

backend/tomato/api/host.py

@@ -169,6 +169,16 @@ def host_modify(name, attrs):
 	host_list.invalidate()
 	return h.info()
 
+@checkauth
+def host_deactivate(name):
+	"""
+	undocumented
+	"""
+	h = _getHost(name)
+	h.deactivate()
+	return h.info()
+
+
 @checkauth
 def host_remove(name):
 	"""
@@ -178,6 +188,7 @@ def host_remove(name):
 	h.remove()
 	host_list.invalidate()
 
+
 @checkauth
 def host_users(name):
 	"""

backend/tomato/config.py

@@ -178,4 +178,11 @@
 A new user, %s, has just registered at the ToMaTo testbed.\n\
 You can review all pending user registrations at https://master.tomato-lab.org/account/registrations\n\n\
 Best Wishes,\nThe ToMaTo Testbed"								
-}
+}
+
+
+
+# Load Balancing, Migration, Resource Allocator:
+MIGRATION_TRESHOLD = 1.2; #New Host has to be 20% better
+AVG_MINIMUM = 0.25;
+AVG_MAXIMUM = 0.7;

backend/tomato/elements/__init__.py

@@ -468,6 +468,8 @@ def fetchInfo(self):
 		mel = self.mainElement()
 		if mel:
 			mel.updateInfo()
+
+
 
 	def updateUsage(self):
 		self.totalUsage.updateFrom([el.usageStatistics for el in self.getHostElements()]

backend/tomato/elements/external_network.py

@@ -19,10 +19,11 @@
 from .. import elements, resources, host
 from ..resources import network as r_network
 from ..lib.attributes import Attr #@UnresolvedImport
-from generic import ST_CREATED, ST_STARTED
+from generic import ST_CREATED, ST_STARTED, ST_PREPARED
 from .. import currentUser
 from ..auth import Flags
 from ..lib.error import UserError
+from tomato.config import MIGRATION_TRESHOLD
 
 class External_Network(elements.generic.ConnectingElement, elements.Element):
 	name_attr = Attr("name", desc="Name")
@@ -36,6 +37,7 @@ class External_Network(elements.generic.ConnectingElement, elements.Element):
 	CUSTOM_ACTIONS = {
 		"start": [ST_CREATED],
 		"stop": [ST_STARTED],
+		"migrate": [ST_PREPARED],
 		elements.REMOVE_ACTION: [ST_CREATED],
 	}
 	CUSTOM_ATTRS = {
@@ -192,6 +194,42 @@ def action_stop(self):
 			self.element = None
 		self.setState(ST_CREATED, True)
 
+	def checkMigrate(self):
+		#We only migrate if the external network is already started, otherwise we don't have to migrate
+		return self.state in [ST_PREPARED]
+
+	def try_migrate(self):
+
+		UserError.check(self.element.checkMigrate(), code=UserError.UNSUPPORTED_ACTION, message="Element can't be migrated",data={"type": self.TYPE})
+
+
+		hPref, sPref = self.getLocationPrefs()
+
+		bestHost,bestPref = host.getBestHost(site=self.site, elementTypes=[self.TYPE]+self.CAP_CHILDREN.keys(), hostPrefs=hPref, sitePrefs=sPref)
+		UserError.check(bestHost, code=UserError.NO_RESOURCES, message="No matching host found for element", data={"type": self.TYPE})
+
+
+		hostScore = host.getHostScore(self.element.host,site=self.site, elementTypes=[self.TYPE]+self.CAP_CHILDREN.keys(), hostPrefs=hPref, sitePrefs=sPref)
+
+		if bestPref > hostScore*MIGRATION_TRESHOLD:
+			return		
+
+
+
+		kind = self.getParent().network.kind if self.parent and self.getParent().samenet else self.kind
+
+		if self.parent and self.getParent().samenet:
+			self.network = r_network.getInstance(host, self.getParent().network.kind)
+		else:
+			self.network = r_network.getInstance(host, self.kind)			
+		attrs = {"network": self.network.network.kind}
+
+		self.action_destroy()
+		self.element = bestHost.createElement("external_network", parent=None, attrs=attrs, ownerElement=self)
+		self.setState(ST_STARTED)
+		self.triggerConnectionStart()
+
+
 	def readyToConnect(self):
 		return bool(self.element)
 

backend/tomato/elements/generic.py

@@ -14,14 +14,16 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>
-
 from django.db import models
 from .. import elements, resources, host
 from ..resources import profile as r_profile, template as r_template
 from ..lib.attributes import Attr, attribute #@UnresolvedImport
-from ..lib import attributes #@UnresolvedImport
+from ..lib import attributes,rpc #@UnresolvedImport
 from ..lib.error import UserError, InternalError
+from ..lib.util import upload
 import time
+import urllib
+from tomato.config import MIGRATION_TRESHOLD
 
 ST_CREATED = "created"
 ST_PREPARED = "prepared"
@@ -47,6 +49,7 @@ class VMElement(elements.Element):
 		"prepare": [ST_CREATED],
 		"destroy": [ST_PREPARED],
 		"change_template": [ST_CREATED, ST_PREPARED],
+		"migrate": [ST_PREPARED],
 		elements.REMOVE_ACTION: [ST_CREATED],
 	}
 	CUSTOM_ATTRS = {
@@ -237,6 +240,47 @@ def after_start(self):
 	def after_upload_use(self):
 		self.custom_template = True
 		self.save()
+
+	def try_migrate(self):
+
+
+		if self.state in [ST_CREATED]: return
+
+		hPref, sPref = self.getLocationPrefs()
+
+		bestHost,bestPref = host.getBestHost(site=self.site, elementTypes=[self.TYPE]+self.CAP_CHILDREN.keys(), hostPrefs=hPref, sitePrefs=sPref)
+		UserError.check(bestHost, code=UserError.NO_RESOURCES, message="No matching host found for element", data={"type": self.TYPE})
+
+		hostScore = host.getHostScore(self.element.host,site=self.site, elementTypes=[self.TYPE]+self.CAP_CHILDREN.keys(), hostPrefs=hPref, sitePrefs=sPref)
+
+		if bestPref > hostScore*MIGRATION_TRESHOLD:
+			return		
+
+
+		#Download template. Receive download_grant from template and save it to a tempfile?
+		urllib.urlretrieve(self.element.host.grantUrl(self.element.action("download_grant"), "download"), self.name+"_tmp_image.tar.gz")
+
+		attrs = self._remoteAttrs()
+		#Create identical element on new host
+		new_el = bestHost.createElement(self.TYPE, parent=None, attrs=attrs, ownerElement=self)
+
+		#Kill old element on old host
+		self.action_destroy()
+
+		#Set new element and save it
+		self.element = new_el
+		self.save()
+
+		for iface in self.getChildren():
+			iface.try_migrate()
+		self.element.action("prepare")
+
+		self.setState(ST_PREPARED, True)
+
+		upload(bestHost.grantUrl(new_el.action("upload_grant"),"upload"),self.name+"_tmp_image.tar.gz")
+		new_el.action("upload_use")	
+
+
 
 	def upcast(self):
 		return self
@@ -297,13 +341,31 @@ def _remove(self):
 			self.element = None
 			self.save()
 
+	def checkMigrate(self):
+		return self.state in [ST_CREATED,ST_PREPARED]
+
+	def try_migrate(self):
+
+		if self.state in [ST_CREATED,ST_STARTED]: return
+
+
+
+		parEl = self.getParent().element
+		assert parEl
+		attrs = self._remoteAttrs()
+		element = parEl.createChild(self.TYPE, attrs=attrs, ownerElement=self)
+
+		self.element.remove()
+		self.element = element
+		self.save()
+
 	def readyToConnect(self):
 		return self.state == ST_STARTED
 
 	def upcast(self):
 		return self
 
-
+		
 class ConnectingElement:
 	def getLocationData(self, maxDepth=3):
 		"""

backend/tomato/elements/repy.py

@@ -58,6 +58,8 @@ def action_destroy(self):
 			self.element = None
 		self.setState(generic.ST_CREATED, True)
 
+
+
 
 class Repy_Interface(generic.VMInterface):
 	TYPE = "repy_interface"

backend/tomato/elements/tinc.py

@@ -22,6 +22,7 @@
 from ..lib.attributes import Attr #@UnresolvedImport
 from generic import ST_CREATED, ST_PREPARED, ST_STARTED
 from ..lib.error import UserError, assert_
+from tomato.config import MIGRATION_TRESHOLD
 
 class Tinc_VPN(elements.generic.ConnectingElement, elements.Element):
 	name_attr = Attr("name", desc="Name", type="str")
@@ -34,6 +35,7 @@ class Tinc_VPN(elements.generic.ConnectingElement, elements.Element):
 		"destroy": [ST_PREPARED],
 		"start": [ST_PREPARED],
 		"stop": [ST_STARTED],
+		"migrate": [ST_PREPARED,ST_STARTED],
 		elements.REMOVE_ACTION: [ST_CREATED],
 	}
 	CUSTOM_ATTRS = {
@@ -225,6 +227,31 @@ def action_start(self):
 		self._parallelChildActions(self._childsByState()[ST_PREPARED], "start")
 		self.setState(ST_STARTED)
 
+
+	def checkMigrate(self):
+		return self._childsByState()[ST_PREPARED] != [] or self._childsByState()[ST_CREATED] != []
+
+	def try_migrate(self):
+
+		childList = self._childsByState()[ST_PREPARED]
+		lock = threading.RLock()
+		class WorkerThread(threading.Thread):
+			def run(self):
+				while True:
+					with lock:
+						if not childList:
+							return
+						ch = childList.pop()
+					ch.try_migrate()
+		threads = []
+		for _ in xrange(0, min(len(childList), 10)):
+			thread = WorkerThread()
+			threads.append(thread)
+			thread.start()
+		for thread in threads:
+			thread.join()
+
+
 	def upcast(self):
 		return self
 
@@ -249,6 +276,7 @@ class Tinc_Endpoint(elements.generic.ConnectingElement, elements.Element):
 		"stop": [ST_STARTED],
 		"prepare": [ST_CREATED],
 		"destroy": [ST_PREPARED],
+		"migrate": [ST_PREPARED,ST_STARTED],
 		elements.REMOVE_ACTION: [ST_CREATED],
 	}
 	CUSTOM_ATTRS = {
@@ -335,6 +363,30 @@ def action_stop(self):
 			self.element.action("stop")
 		self.setState(ST_PREPARED, True)
 
+	def checkMigrate(self):		
+		return self.state in [ST_CREATED,ST_PREPARED]
+
+	def try_migrate(self):
+
+		if self.state in [ST_CREATED]: return
+
+		hPref, sPref = self.getLocationPrefs()
+
+
+		bestHost,bestPref = host.getBestHost(elementTypes=["tinc"], hostPrefs=hPref, sitePrefs=sPref)
+		UserError.check(bestHost, code=UserError.NO_RESOURCES, message="No matching host found for element", data={"type": self.TYPE})
+
+		hostScore = host.getHostScore(self.element.host,elementTypes=["tinc"], hostPrefs=hPref, sitePrefs=sPref)
+
+		if bestPref > hostScore*MIGRATION_TRESHOLD:
+			return		
+
+
+		self.action_destroy()
+		self.element = bestHost.createElement(self.remoteType(), parent=None, attrs=attrs, ownerElement=self)
+		self.save()
+		self.setState(ST_PREPARED, True)	
+
 	def upcast(self):
 		return self
 

backend/tomato/elements/udp.py

@@ -20,6 +20,8 @@
 from ..lib.attributes import Attr #@UnresolvedImport
 from generic import ST_CREATED, ST_PREPARED, ST_STARTED
 from ..lib.error import UserError
+from tomato.config import MIGRATION_TRESHOLD
+
 
 class UDP_Endpoint(elements.Element):
 	element = models.ForeignKey(host.HostElement, null=True, on_delete=models.SET_NULL)
@@ -32,6 +34,7 @@ class UDP_Endpoint(elements.Element):
 		"stop": [ST_STARTED],
 		"prepare": [ST_CREATED],
 		"destroy": [ST_PREPARED],
+		"migrate": [ST_PREPARED,ST_STARTED],
 		elements.REMOVE_ACTION: [ST_CREATED],
 	}
 	CUSTOM_ATTRS = {
@@ -111,6 +114,30 @@ def action_stop(self):
 			self.element.action("stop")
 		self.setState(ST_PREPARED, True)
 
+	def checkMigrate(self):
+		return self.state in [ST_CREATED,ST_PREPARED]
+
+
+	def try_migrate(self):		
+
+		if self.state in [ST_CREATED]: return
+
+		host_ = host.select(elementTypes=[self.remoteType()])
+
+		bestHost,bestPref = host.getBestHost(elementTypes=[self.remoteType()])
+		UserError.check(bestHost, code=UserError.NO_RESOURCES, message="No matching host found for element", data={"type": self.TYPE})
+
+		hostScore = host.getHostScore(self.element.host,elementTypes=[self.remoteType()])
+
+		if bestPref > hostScore*MIGRATION_TRESHOLD:
+			return		
+
+
+		self.action_destroy()
+		self.element = bestHost.createElement(self.remoteType(), parent=None, attrs=attrs, ownerElement=self)
+		self.save()
+		self.setState(ST_PREPARED, True)
+
 	def upcast(self):
 		return self