Skip to content
This repository has been archived by the owner on Jun 27, 2023. It is now read-only.

Commit

Permalink
closes #7: reduce memory usage using soft refs
Browse files Browse the repository at this point in the history
  • Loading branch information
@bblfish
bblfish committed Nov 13, 2013
1 parent 7939ebe commit 8e7491d
Show file tree
Hide file tree
Showing 2 changed files with 74 additions and 90 deletions.
79 changes: 31 additions & 48 deletions app/rww/ldp/PlantainLDPCActor.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ import scala.io.Codec
import java.io.{File, FileOutputStream}
import java.nio.file.DirectoryStream.Filter
import org.w3.banana.diesel.Diesel
import com.google.common.cache._
import java.util.concurrent.Callable
import scalax.io.Resource

/**
* A LDP Container actor that is responsible for the equivalent of a directory
Expand All @@ -37,10 +40,6 @@ class PlantainLDPCActor(baseUri: Plantain#URI, root: Path)
import ops._
import diesel._





override lazy val fileName = ""

override def preStart {
Expand Down Expand Up @@ -74,7 +73,6 @@ class PlantainLDPCActor(baseUri: Plantain#URI, root: Path)
})
}


override
def localName(uri: Plantain#URI): String = {
val requestedPath = uri.underlying.getPath
Expand All @@ -84,7 +82,6 @@ class PlantainLDPCActor(baseUri: Plantain#URI, root: Path)
else super.localName(uri)
}


private def descriptionFor(path: Path, attrs: BasicFileAttributes): Plantain#Graph = {
def graphFor(uri: Plantain#URI) = {
var res = emptyGraph +
Expand Down Expand Up @@ -338,7 +335,32 @@ class PlantainLDPRActor(val baseUri: Plantain#URI,path: Path)
import ops._
import scala.collection.convert.decorateAsScala._

val cache = scala.collection.mutable.Map[String, NamedResource[Plantain]]()
//google cache with soft values: at least it will remove the simplest failures
val cacheg: LoadingCache[String,NamedResource[Plantain]] = CacheBuilder.newBuilder()
.softValues()
.build(new CacheLoader[String,NamedResource[Plantain]]() {
import scalax.io.{Resource=>xResource}

def load(key: String) = {
//at this point it is still very easy - only two cases! but won't stay like this...
val (file, iri) = fileAndURIFor(key)

if (file.createNewFile()) {
if (file.toString.endsWith(ext)) {
LocalLDPR[Plantain](iri, Graph.empty, Option(new Date(path.toFile.lastModified())))
} else {
LocalBinaryR[Plantain](file.toPath, iri)
}
} else {
val res = xResource.fromFile(file)
val ldprOpt = reader.read(res, iri.toString).map {
g =>
LocalLDPR[Plantain](iri, g, Option(new Date(path.toFile.lastModified())))
}
ldprOpt.get //todo: this could break
}
}
})

def filter = new Filter[Path]() {
val fileName = path.getFileName.toString
Expand Down Expand Up @@ -374,48 +396,9 @@ class PlantainLDPRActor(val baseUri: Plantain#URI,path: Path)
*/
@throws[ResourceDoesNotExist]
def getResource(name: String): NamedResource[Plantain] = {
import scalax.file.{Path=>xPath}
import scalax.io.{Resource=>xResource}
//todo: the file should be verified to see if it is up to date.
cache.getOrElseUpdate(name, {
//at this point it is still very easy - only two cases! but won't stay like this...
val (file,iri) = fileAndURIFor(name)

if (file.createNewFile()) {
if (file.toString.endsWith(ext)) {
LocalLDPR[Plantain](iri,Graph.empty,Option(new Date(path.toFile.lastModified())))
} else {
LocalBinaryR[Plantain](file.toPath, iri)
}
} else {
val res = xResource.fromFile(file)
val ldprOpt = reader.read(res, iri.toString).map { g =>
LocalLDPR[Plantain](iri, g, Option(new Date(path.toFile.lastModified())))
}
ldprOpt.get //todo: this could break
}
})

// //todo: this globbing is not secure. Requests could contain files with globs in them.
// val pathStream = Files.newDirectoryStream(path.getParent,fileName+".*")
// val paths = try {
// for (p <- pathStream.asScala) yield p
// } finally {
// pathStream.close()
// }
// paths.collectFirst{
// case path if path.endsWith(".ttl") =>
// val res = xResource.fromFile(path.toFile)
// cache = reader.read(res,"").toOption.map{g=>
// LocalLDPR[Plantain](baseUri, g,Option(new Date(path.toFile.lastModified())))
// }
// cache
// case path if path.endsWith(".jpg") => {
// cache = Option(LocalBinaryR[Plantain](path,baseUri))
// cache
// }
// }.flatten
// }
cacheg.get(name)
}

/**
Expand Down Expand Up @@ -451,7 +434,7 @@ class PlantainLDPRActor(val baseUri: Plantain#URI,path: Path)
case scala.util.Failure(t) => throw new StoreProblem(t)
case x => x
}
cache.put(name,LocalLDPR[Plantain](iri,graph,Some(new Date(file.lastModified()))))
cacheg.put(name,LocalLDPR[Plantain](iri,graph,Some(new Date(file.lastModified()))))
}


Expand Down
85 changes: 43 additions & 42 deletions project/Build.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,57 +4,58 @@ import play.Project._

object ApplicationBuild extends Build {

val appName = "RWWeb"
val appVersion = "0.7.2-SNAPSHOT"
val appName = "RWWeb"
val appVersion = "0.7.2-SNAPSHOT"

val banana = (name:String) => "org.w3" %% name % "2013_10_07-SNAPSHOT" excludeAll(ExclusionRule(organization = "org.scala-stm"))
val banana = (name: String) => "org.w3" %% name % "2013_10_07-SNAPSHOT" excludeAll (ExclusionRule(organization = "org.scala-stm"))

val iterateeDeps = "com.typesafe.play" %% "play-iteratees" % "2.2.0"
val scalatest = "org.scalatest" %% "scalatest" % "2.0.RC1-SNAP4"
val scalaActors = "org.scala-lang" % "scala-actors" % "2.10.2"
val iterateeDeps = "com.typesafe.play" %% "play-iteratees" % "2.2.0"
val scalatest = "org.scalatest" %% "scalatest" % "2.0.RC1-SNAP4"
val scalaActors = "org.scala-lang" % "scala-actors" % "2.10.2"

val testsuiteDeps =
Seq(
// scalaActors,
scalatest
val testsuiteDeps =
Seq(
// scalaActors,
scalatest
)


val appDependencies = Seq("banana-sesame","banana-jena","banana-rdf","plantain").map(banana) ++ Seq(
"net.rootdev" % "java-rdfa" % "0.4.2-RC2",
"nu.validator.htmlparser" % "htmlparser" % "1.2.1",
"org.scalaz" % "scalaz-core_2.10" % "7.0.0-RC1", // from "http://repo.typesafe.com/typesafe/releases/org/scalaz/scalaz-core_2.10.0-M6/7.0.0-M2/scalaz-core_2.10.0-M6-7.0.0-M2.jar"
"org.bouncycastle" % "bcprov-jdk15on" % "1.47",
"org.scala-lang" % "scala-actors" % "2.10.0", //for tests because of sbt for some reason
"net.sf.uadetector" % "uadetector-resources" % "2012.12",
iterateeDeps,
"org.scalatest" %% "scalatest" % "2.0.RC1-SNAP4",
"org.scala-lang" % "scala-actors" % "2.10.2"

// "com.typesafe" %% "play-mini" % "2.0.1",
)

val appDependencies = Seq("banana-sesame", "banana-jena", "banana-rdf", "plantain").map(banana) ++ Seq(
"net.rootdev" % "java-rdfa" % "0.4.2-RC2",
"nu.validator.htmlparser" % "htmlparser" % "1.2.1",
"org.scalaz" % "scalaz-core_2.10" % "7.0.0-RC1", // from "http://repo.typesafe.com/typesafe/releases/org/scalaz/scalaz-core_2.10.0-M6/7.0.0-M2/scalaz-core_2.10.0-M6-7.0.0-M2.jar"
"org.bouncycastle" % "bcprov-jdk15on" % "1.47",
"org.scala-lang" % "scala-actors" % "2.10.0", //for tests because of sbt for some reason
"net.sf.uadetector" % "uadetector-resources" % "2012.12",
iterateeDeps,
"org.scalatest" %% "scalatest" % "2.0.RC1-SNAP4",
"org.scala-lang" % "scala-actors" % "2.10.2",
"com.google.guava" % "guava" % "15.0",
"com.google.code.findbugs" % "jsr305" % "2.0.2"

// "com.typesafe" %% "play-mini" % "2.0.1",
)


val main = play.Project(appName, appVersion, appDependencies).settings(
resolvers += "Sonatype snapshots" at "http://oss.sonatype.org/content/repositories/snapshots", //for latest scalaz
resolvers += "sesame-repo-releases" at "http://repo.aduna-software.org/maven2/releases",
/*
if you want to compile banana-* yourself then you need to remove the following resolver
you may need to first remove the org.w3 directory from the Play/repository/cache directory
note: you can use `publish_local` from you banana-rdf clone but must start sbt with
$ sbt -ivy $RWW_PLAY_HOME/Play20/repository/
finally you may need to rebuild your IDE files ( clearing the previous ones perhaps )
*/
resolvers += "bblfish-snapshots" at "http://bblfish.net/work/repo/snapshots",
scalaVersion := "2.10.2",
javacOptions ++= Seq("-source","1.7", "-target","1.7"),
initialize := {
//thanks to http://stackoverflow.com/questions/19208942/enforcing-java-version-for-scala-project-in-sbt/19271814?noredirect=1#19271814
val _ = initialize.value // run the previous initialization
val specVersion = sys.props("java.specification.version")
assert(java.lang.Float.parseFloat(specVersion) >= 1.7, "Java 1.7 or above required. Your version is " + specVersion)
}
resolvers += "Sonatype snapshots" at "http://oss.sonatype.org/content/repositories/snapshots", //for latest scalaz
resolvers += "sesame-repo-releases" at "http://repo.aduna-software.org/maven2/releases",
/*
if you want to compile banana-* yourself then you need to remove the following resolver
you may need to first remove the org.w3 directory from the Play/repository/cache directory
note: you can use `publish_local` from you banana-rdf clone but must start sbt with
$ sbt -ivy $RWW_PLAY_HOME/Play20/repository/
finally you may need to rebuild your IDE files ( clearing the previous ones perhaps )
*/
resolvers += "bblfish-snapshots" at "http://bblfish.net/work/repo/snapshots",
scalaVersion := "2.10.2",
javacOptions ++= Seq("-source", "1.7", "-target", "1.7"),
initialize := {
//thanks to http://stackoverflow.com/questions/19208942/enforcing-java-version-for-scala-project-in-sbt/19271814?noredirect=1#19271814
val _ = initialize.value // run the previous initialization
val specVersion = sys.props("java.specification.version")
assert(java.lang.Float.parseFloat(specVersion) >= 1.7, "Java 1.7 or above required. Your version is " + specVersion)
}


)
Expand Down

0 comments on commit 8e7491d

Please sign in to comment.