Skip to content

Commit

Permalink
Backport of S3 bulk delete bug fix
Browse files Browse the repository at this point in the history
  • Loading branch information
KarlSjostrand committed Feb 11, 2019
1 parent 8694b45 commit 005e37a
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 5 deletions.
2 changes: 1 addition & 1 deletion build.sbt
@@ -1,6 +1,6 @@

name := "slicebox"
version := "1.6"
version := "1.6.1"
organization := "se.nimsa"
scalaVersion := "2.12.7"
scalacOptions := Seq("-encoding", "UTF-8", "-Xlint", "-deprecation", "-unchecked", "-feature", "-target:jvm-1.8")
Expand Down
9 changes: 5 additions & 4 deletions src/main/scala/se/nimsa/sbx/storage/S3Storage.scala
Expand Up @@ -65,15 +65,16 @@ class S3Storage(val bucket: String, val s3Prefix: String, val region: String)(im
s3.deleteObject(bucket, sourceImageName)
}

override def deleteByName(names: Seq[String]): Unit =
if (names.lengthCompare(1) == 0)
override def deleteByName(names: Seq[String]): Unit = {
if (names.lengthCompare(1) == 0) {
s3.deleteObject(bucket, s3Id(names.head))
else {
} else {
// micro-batch this since S3 accepts up to 1000 deletes at a time
names.grouped(1000).map { subset =>
names.grouped(1000).foreach { subset =>
s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(subset.map(name => s3Id(name)): _*).withQuiet(true))
}
}
}

override def fileSink(name: String)(implicit executionContext: ExecutionContext): Sink[ByteString, Future[Done]] =
Sink
Expand Down

0 comments on commit 005e37a

Please sign in to comment.