cyclane
/
teamcity-executors-test-task
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Releases 2 Activity

CLI implementation & linting
Test Workflow / Lint and test library (pull_request) Successful in 8m18s Details

This commit is contained in:
Gleb Koval 2023-12-29 23:33:00 +06:00
parent 36728fa1e9
commit 0e41b2ab8a
Signed by: cyclane
GPG Key ID: 15E168A8B332382C
16 changed files with 563 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.editorconfig Normal file
View File

@ -0,0 +1,16 @@
root = true
[*]
charset = utf-8
end_of_line = lf
indent_size = 4
indent_style = space
insert_final_newline = false
max_line_length = 120
tab_width = 4
[{*.yaml,*.yml}]
indent_size = 2
[{*.kt,*.kts}]
ij_kotlin_packages_to_use_import_on_demand = org.junit.jupiter.api,aws.sdk.kotlin.services.s3,kotlinx.coroutines,java.io,ziputils

124
.idea/uiDesigner.xml Normal file
View File

@ -0,0 +1,124 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Palette2">
<group name="Swing">
<item class="com.intellij.uiDesigner.HSpacer" tooltip-text="Horizontal Spacer" icon="/com/intellij/uiDesigner/icons/hspacer.svg" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="1" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="com.intellij.uiDesigner.VSpacer" tooltip-text="Vertical Spacer" icon="/com/intellij/uiDesigner/icons/vspacer.svg" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="1" anchor="0" fill="2" />
</item>
<item class="javax.swing.JPanel" icon="/com/intellij/uiDesigner/icons/panel.svg" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3" />
</item>
<item class="javax.swing.JScrollPane" icon="/com/intellij/uiDesigner/icons/scrollPane.svg" removable="false" auto-create-binding="false" can-attach-label="true">
<default-constraints vsize-policy="7" hsize-policy="7" anchor="0" fill="3" />
</item>
<item class="javax.swing.JButton" icon="/com/intellij/uiDesigner/icons/button.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="0" fill="1" />
<initial-values>
<property name="text" value="Button" />
</initial-values>
</item>
<item class="javax.swing.JRadioButton" icon="/com/intellij/uiDesigner/icons/radioButton.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="RadioButton" />
</initial-values>
</item>
<item class="javax.swing.JCheckBox" icon="/com/intellij/uiDesigner/icons/checkBox.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="CheckBox" />
</initial-values>
</item>
<item class="javax.swing.JLabel" icon="/com/intellij/uiDesigner/icons/label.svg" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="8" fill="0" />
<initial-values>
<property name="text" value="Label" />
</initial-values>
</item>
<item class="javax.swing.JTextField" icon="/com/intellij/uiDesigner/icons/textField.svg" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JPasswordField" icon="/com/intellij/uiDesigner/icons/passwordField.svg" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JFormattedTextField" icon="/com/intellij/uiDesigner/icons/formattedTextField.svg" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JTextArea" icon="/com/intellij/uiDesigner/icons/textArea.svg" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTextPane" icon="/com/intellij/uiDesigner/icons/textPane.svg" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JEditorPane" icon="/com/intellij/uiDesigner/icons/editorPane.svg" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JComboBox" icon="/com/intellij/uiDesigner/icons/comboBox.svg" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="2" anchor="8" fill="1" />
</item>
<item class="javax.swing.JTable" icon="/com/intellij/uiDesigner/icons/table.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JList" icon="/com/intellij/uiDesigner/icons/list.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="2" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTree" icon="/com/intellij/uiDesigner/icons/tree.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTabbedPane" icon="/com/intellij/uiDesigner/icons/tabbedPane.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSplitPane" icon="/com/intellij/uiDesigner/icons/splitPane.svg" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSpinner" icon="/com/intellij/uiDesigner/icons/spinner.svg" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSlider" icon="/com/intellij/uiDesigner/icons/slider.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSeparator" icon="/com/intellij/uiDesigner/icons/separator.svg" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3" />
</item>
<item class="javax.swing.JProgressBar" icon="/com/intellij/uiDesigner/icons/progressbar.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="javax.swing.JToolBar" icon="/com/intellij/uiDesigner/icons/toolbar.svg" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1">
<preferred-size width="-1" height="20" />
</default-constraints>
</item>
<item class="javax.swing.JToolBar$Separator" icon="/com/intellij/uiDesigner/icons/toolbarSeparator.svg" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="0" fill="1" />
</item>
<item class="javax.swing.JScrollBar" icon="/com/intellij/uiDesigner/icons/scrollbar.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="0" anchor="0" fill="2" />
</item>
</group>
</component>
</project>

58
README.md
View File

@ -6,6 +6,64 @@ an AWS S3 bucket.
This tool is released as a JAR in the [release page](https://git.koval.net/cyclane/teamcity-executors-test-task/releases). This tool is released as a JAR in the [release page](https://git.koval.net/cyclane/teamcity-executors-test-task/releases).
Use `java -jar <backup-jar-name>.jar --help` for more detailed usage instructions. Use `java -jar <backup-jar-name>.jar --help` for more detailed usage instructions.
### --help
```
Usage: s3backup-tool [<options>] <command> [<args>]...
A simple AWS S3 backup tool. This tool assumes credentials are properly configured using aws-cli.
Options:
-h, --help Show this message and exit
Commands:
create Create a backup of a file or directory.
restore Restore a backup from AWS S3.
restore-file Restore a single file from a backup from AWS S3.
```
#### Subcommands
```
Usage: s3backup-tool create [<options>] <source> <bucket>
Create a backup of a file or directory.
Options:
-h, --help Show this message and exit
Arguments:
<source> File or directory to backup
<bucket> Name of S3 bucket to backup to
```
```
Usage: s3backup-tool restore [<options>] <bucket> <backupkey> <destination>
Restore a backup from AWS S3.
Options:
-h, --help Show this message and exit
Arguments:
<bucket> Name of S3 bucket to restore the backup from
<backupkey> The S3 key of the backup to restore
<destination> Directory to restore to
```
```
Usage: s3backup-tool restore-file [<options>] <bucket> <backupkey> <filepath> <destination>
Restore a single file from a backup from AWS S3.
Options:
-h, --help Show this message and exit
Arguments:
<bucket> Name of S3 bucket to restore the backup from
<backupkey> The S3 key of the backup to restore
<filepath> File path within the backup
<destination> Directory to restore to
```
## Assumptions ## Assumptions
1. This test task is not interested in re-implementations of common libraries (AWS SDK, Clikt, Gradle Shadow, ...) 1. This test task is not interested in re-implementations of common libraries (AWS SDK, Clikt, Gradle Shadow, ...)
2. The last part (restoration of a single file) should be optimised so that only the part of the blob required for this 2. The last part (restoration of a single file) should be optimised so that only the part of the blob required for this

11
build.gradle.kts
View File

@ -1,7 +1,7 @@
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
plugins { plugins {
application
kotlin("jvm") version "1.9.21" kotlin("jvm") version "1.9.21"
id("org.jlleitschuh.gradle.ktlint") version "12.0.3"
id("com.github.johnrengelman.shadow") version "8.1.1" id("com.github.johnrengelman.shadow") version "8.1.1"
} }
@ -16,6 +16,7 @@ dependencies {
implementation("aws.sdk.kotlin:s3:1.0.25") implementation("aws.sdk.kotlin:s3:1.0.25")
implementation("org.slf4j:slf4j-simple:2.0.9") implementation("org.slf4j:slf4j-simple:2.0.9")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3")
implementation("com.github.ajalt.clikt:clikt:4.2.1")
testImplementation("org.jetbrains.kotlin:kotlin-test") testImplementation("org.jetbrains.kotlin:kotlin-test")
} }
@ -25,8 +26,6 @@ tasks.test {
kotlin { kotlin {
jvmToolchain(17) jvmToolchain(17)
} }
tasks.jar { application {
manifest { mainClass.set("MainKt")
attributes("Main-Class" to "backup.MainKt")
}
} }

3
settings.gradle.kts
View File

@ -1,5 +1,4 @@
plugins { plugins {
id("org.gradle.toolchains.foojay-resolver-convention") version "0.5.0" id("org.gradle.toolchains.foojay-resolver-convention") version "0.5.0"
} }
rootProject.name = "teamcity-executors-test-task" rootProject.name = "teamcity-executors-test-task"

80
src/main/kotlin/Main.kt Normal file
View File

@ -0,0 +1,80 @@
import aws.sdk.kotlin.services.s3.S3Client
import backup.BackupClient
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.core.subcommands
import com.github.ajalt.clikt.parameters.arguments.argument
import com.github.ajalt.clikt.parameters.arguments.help
import com.github.ajalt.clikt.parameters.types.file
import kotlinx.coroutines.runBlocking
import kotlin.system.exitProcess
fun main(args: Array<String>) =
runBlocking {
S3Client.fromEnvironment().use { s3 ->
S3BackupTool()
.subcommands(
Create(s3),
Restore(s3),
RestoreFile(s3),
)
.main(args)
}
}
class S3BackupTool : CliktCommand(
help = "A simple AWS S3 backup tool. This tool assumes credentials are properly configured using aws-cli.",
) {
override fun run() {
shortHelp(currentContext)
}
}
class Create(val s3: S3Client) : CliktCommand(
help = "Create a backup of a file or directory.",
) {
val source by argument().file(mustExist = true).help("File or directory to backup")
val bucket by argument().help("Name of S3 bucket to backup to")
override fun run() =
runBlocking {
val backupKey = BackupClient(s3, bucket).upload(source)
echo("Successfully created backup with key '$backupKey'")
}
}
class Restore(val s3: S3Client) : CliktCommand(
help = "Restore a backup from AWS S3.",
) {
val bucket by argument().help("Name of S3 bucket to restore the backup from")
val backupKey by argument().help("The S3 key of the backup to restore")
val destination by argument().file(mustExist = true).help("Directory to restore to")
override fun run() =
runBlocking {
if (!destination.isDirectory) {
echo("Destination must be an existing directory", err = true)
exitProcess(1)
}
BackupClient(s3, bucket).restore(destination.toPath(), backupKey)
echo("Successfully restored backup '$backupKey' to '$destination'")
}
}
class RestoreFile(val s3: S3Client) : CliktCommand(
help = "Restore a single file from a backup from AWS S3.",
) {
val bucket by argument().help("Name of S3 bucket to restore the backup from")
val backupKey by argument().help("The S3 key of the backup to restore")
val filePath by argument().help("File path within the backup")
val destination by argument().file(mustExist = true).help("Directory to restore to")
override fun run() =
runBlocking {
if (!destination.isDirectory) {
echo("Destination must be an existing directory", err = true)
exitProcess(1)
}
BackupClient(s3, bucket).restoreFile(destination.toPath(), backupKey, filePath)
echo("Successfully restored '$filePath' from backup '$backupKey' to '$destination'")
}
}

326
src/main/kotlin/backup/BackupClient.kt
View File

@ -30,89 +30,98 @@ import kotlin.io.path.createDirectory
class BackupClient( class BackupClient(
private val s3: S3Client, private val s3: S3Client,
private val bucketName: String, private val bucketName: String,
private val bufSize: Int = 1024 * 1024 * 100 private val bufSize: Int = 1024 * 1024 * 32,
) { ) {
/** /**
* Upload a file/directory backup to AWS S3. * Upload a file/directory backup to AWS S3.
* @param file The File object for the file or directory. * @param file The File object for the file or directory.
*/ */
suspend fun upload(file: File) = coroutineScope { suspend fun upload(file: File) =
val backupKey = "${file.name}/${Instant.now()}.zip" coroutineScope {
PipedInputStream().use { inputStream -> val backupKey = "${file.canonicalFile.name}/${Instant.now()}.zip"
val outputStream = PipedOutputStream(inputStream) PipedInputStream().use { inputStream ->
val zipper = launch(Dispatchers.IO) { val outputStream = PipedOutputStream(inputStream)
file.compressToZip(outputStream) val zipper =
} launch(Dispatchers.IO) {
file.compressToZip(outputStream)
}
val data = ByteArray(bufSize) val data = ByteArray(bufSize)
val initialRead = inputStream.readNBytes(data, 0, bufSize) val initialRead = inputStream.readNBytes(data, 0, bufSize)
if (initialRead == bufSize) { if (initialRead == bufSize) {
// Large upload, use multipart // Large upload, use multipart
// TODO: multipart uploads can be asynchronous, which would improve // TODO: multipart uploads can be asynchronous, which would improve
// performance a little bit for big uploads. // performance a little bit for big uploads.
val upload = s3.createMultipartUpload { val upload =
bucket = bucketName s3.createMultipartUpload {
key = backupKey
}
try {
val uploadParts = mutableListOf<CompletedPart>()
var number = 1
var bytesRead = initialRead
while (bytesRead > 0) {
val part = s3.uploadPart {
bucket = bucketName bucket = bucketName
key = backupKey key = backupKey
partNumber = number
uploadId = upload.uploadId
body = ByteStream.fromBytes(data.take(bytesRead))
}.toCompletedPart(number)
uploadParts.add(part)
number++
bytesRead = inputStream.readNBytes(data, 0, bufSize)
}
s3.completeMultipartUpload {
bucket = bucketName
key = backupKey
uploadId = upload.uploadId
multipartUpload = CompletedMultipartUpload {
parts = uploadParts
} }
try {
val uploadParts = mutableListOf<CompletedPart>()
var number = 1
var bytesRead = initialRead
while (bytesRead > 0) {
val part =
s3.uploadPart {
bucket = bucketName
key = backupKey
partNumber = number
uploadId = upload.uploadId
body = ByteStream.fromBytes(data.take(bytesRead))
}.toCompletedPart(number)
uploadParts.add(part)
number++
bytesRead = inputStream.readNBytes(data, 0, bufSize)
}
s3.completeMultipartUpload {
bucket = bucketName
key = backupKey
uploadId = upload.uploadId
multipartUpload =
CompletedMultipartUpload {
parts = uploadParts
}
}
} catch (e: Exception) {
s3.abortMultipartUpload {
bucket = bucketName
key = backupKey
uploadId = upload.uploadId
}
throw e
} }
} catch (e: Exception) { } else {
s3.abortMultipartUpload { // Small upload, use single request
s3.putObject {
bucket = bucketName bucket = bucketName
key = backupKey key = backupKey
uploadId = upload.uploadId body = ByteStream.fromBytes(data.take(initialRead))
} }
throw e
}
} else {
// Small upload, use single request
s3.putObject {
bucket = bucketName
key = backupKey
body = ByteStream.fromBytes(data.take(initialRead))
} }
zipper.join() // Should be instant
} }
zipper.join() // Should be instant backupKey
} }
backupKey
}
/** /**
* Restore a backup from AWS S3. * Restore a backup from AWS S3.
* @param destination The destination directory path for the backup contents. * @param destination The destination directory path for the backup contents.
* @param backupKey The S3 key of the backup. * @param backupKey The S3 key of the backup.
*/ */
suspend fun restore(destination: Path, backupKey: String) = coroutineScope { suspend fun restore(
val req = GetObjectRequest { destination: Path,
bucket = bucketName backupKey: String,
key = backupKey ) = coroutineScope {
} val req =
GetObjectRequest {
bucket = bucketName
key = backupKey
}
s3.getObject(req) { resp -> s3.getObject(req) { resp ->
ZipInputStream( ZipInputStream(
resp.body?.toInputStream() resp.body?.toInputStream()
?: throw IOException("S3 response is missing body") ?: throw IOException("S3 response is missing body"),
).use { zipStream -> ).use { zipStream ->
zipStream.decompress { destination.resolve(it) } zipStream.decompress { destination.resolve(it) }
} }
@ -125,81 +134,99 @@ class BackupClient(
* @param backupKey The S3 key of the backup. * @param backupKey The S3 key of the backup.
* @param fileName The full name of the file to restore (including directories if it was under a subdirectory). * @param fileName The full name of the file to restore (including directories if it was under a subdirectory).
*/ */
suspend fun restoreFile(destination: Path, backupKey: String, fileName: String) = coroutineScope { suspend fun restoreFile(
destination: Path,
backupKey: String,
fileName: String,
) = coroutineScope {
// For byte ranges refer to https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.9.TXT // For byte ranges refer to https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.9.TXT
val eocdReq = GetObjectRequest { val eocdReq =
bucket = bucketName GetObjectRequest {
key = backupKey
// Assumption: EOCD has an empty comment
// Assumption: Backups are at least 22 + 20 (= 42) bytes. Only COMPLETELY empty backups can be smaller,
// in which case this function would error anyway, so it should be fine to have this edge-case.
range = "bytes=-${EndOfCentralDirectoryRecord.SIZE + EndOfCentralDirectoryLocator.SIZE}"
}
val eocdBytes = s3.getObject(eocdReq) { resp ->
val bytes = resp.body?.toByteArray() ?: throw IOException("S3 response is missing body")
bytes
}
val eocd = EndOfCentralDirectoryRecord.fromByteArray(eocdBytes, EndOfCentralDirectoryLocator.SIZE)
val eocd64 = if (eocd.eocd64Required()) {
val locator = EndOfCentralDirectoryLocator.fromByteArray(eocdBytes, 0)
val eocd64Req = GetObjectRequest {
bucket = bucketName bucket = bucketName
key = backupKey key = backupKey
range = "bytes=${locator.endOfCentralDirectory64Offset}-" // Assumption: EOCD has an empty comment
// Assumption: Backups are at least 22 + 20 (= 42) bytes. Only COMPLETELY empty backups can be smaller,
// in which case this function would error anyway, so it should be fine to have this edge-case.
range = "bytes=-${EndOfCentralDirectoryRecord.SIZE + EndOfCentralDirectoryLocator.SIZE}"
} }
s3.getObject(eocd64Req) { resp -> val eocdBytes =
s3.getObject(eocdReq) { resp ->
val bytes = resp.body?.toByteArray() ?: throw IOException("S3 response is missing body") val bytes = resp.body?.toByteArray() ?: throw IOException("S3 response is missing body")
EndOfCentralDirectoryRecord64.fromByteArray(bytes, 0) bytes
} }
} else null val eocd = EndOfCentralDirectoryRecord.fromByteArray(eocdBytes, EndOfCentralDirectoryLocator.SIZE)
val cenOffset = if (eocd.centralDirectoryOffset == 0xffffffffU && eocd64 != null) { val eocd64 =
eocd64.centralDirectoryOffset if (eocd.eocd64Required()) {
} else eocd.centralDirectoryOffset.toULong() val locator = EndOfCentralDirectoryLocator.fromByteArray(eocdBytes, 0)
val censReq = GetObjectRequest { val eocd64Req =
bucket = bucketName GetObjectRequest {
key = backupKey bucket = bucketName
// We only know where to fetch until if we've also fetched EOCD64 (which isn't always the case). key = backupKey
// So just over-fetch a little bit, these headers aren't that big anyway. range = "bytes=${locator.endOfCentralDirectory64Offset}-"
range = "bytes=${cenOffset}-" }
} s3.getObject(eocd64Req) { resp ->
val cen = s3.getObject(censReq) { resp -> val bytes = resp.body?.toByteArray() ?: throw IOException("S3 response is missing body")
val bytes = resp.body?.toByteArray() ?: throw IOException("S3 response is missing body") EndOfCentralDirectoryRecord64.fromByteArray(bytes, 0)
var p = 0
while (p < bytes.size) {
try {
val cen = CentralDirectoryFileHeader.fromByteArray(bytes, p)
p += cen.size
if (cen.fileName == fileName) return@getObject cen
} catch (_: InvalidSignatureException) {
return@getObject null
} }
} else {
null
} }
null val cenOffset =
} ?: throw FileNotFoundException("File '${fileName}' not found in backup") if (eocd.centralDirectoryOffset == 0xffffffffU && eocd64 != null) {
eocd64.centralDirectoryOffset
} else {
eocd.centralDirectoryOffset.toULong()
}
val censReq =
GetObjectRequest {
bucket = bucketName
key = backupKey
// We only know where to fetch until if we've also fetched EOCD64 (which isn't always the case).
// So just over-fetch a little bit, these headers aren't that big anyway.
range = "bytes=$cenOffset-"
}
val cen =
s3.getObject(censReq) { resp ->
val bytes = resp.body?.toByteArray() ?: throw IOException("S3 response is missing body")
var p = 0
while (p < bytes.size) {
try {
val cen = CentralDirectoryFileHeader.fromByteArray(bytes, p)
p += cen.size
if (cen.fileName == fileName) return@getObject cen
} catch (_: InvalidDataException) {
return@getObject null
}
}
null
} ?: throw FileNotFoundException("File '$fileName' not found in backup")
val localHeaderOffset = cen.extraFieldRecords.firstNotNullOfOrNull { val localHeaderOffset =
if (it is Zip64ExtraFieldRecord && it.localHeaderOffset != null) it else null cen.extraFieldRecords.firstNotNullOfOrNull {
}?.localHeaderOffset ?: cen.localHeaderOffset.toULong() if (it is Zip64ExtraFieldRecord && it.localHeaderOffset != null) it else null
val compressedSize = cen.extraFieldRecords.firstNotNullOfOrNull { }?.localHeaderOffset ?: cen.localHeaderOffset.toULong()
if (it is Zip64ExtraFieldRecord && it.compressedSize != null) it else null val compressedSize =
}?.compressedSize ?: cen.compressedSize.toULong() cen.extraFieldRecords.firstNotNullOfOrNull {
val req = GetObjectRequest { if (it is Zip64ExtraFieldRecord && it.compressedSize != null) it else null
bucket = bucketName }?.compressedSize ?: cen.compressedSize.toULong()
key = backupKey val req =
range = "bytes=${localHeaderOffset}-${ GetObjectRequest {
// Add CEN min size (46 bytes) so that the next CEN / LOC header is seen by the ZipInputStream bucket = bucketName
// and so it can see the current entry has stopped. key = backupKey
// Note: yes ZipInputStream should know the exact content length from the LOC, but it was still sending range = "bytes=$localHeaderOffset-${
// EOF errors. Perhaps due to fetching multiples of a power of two, or something else. But this helps. // Add CEN min size (46 bytes) so that the next CEN / LOC header is seen by the ZipInputStream
localHeaderOffset + cen.size.toULong() + compressedSize + CentralDirectoryFileHeader.SIZE.toULong() // and so it can see the current entry has stopped.
}" // Note: yes ZipInputStream should know the exact content length from the LOC, but it was still sending
} // EOF errors. Perhaps due to fetching multiples of a power of two, or something else. But this helps.
localHeaderOffset + cen.size.toULong() + compressedSize + CentralDirectoryFileHeader.SIZE.toULong()
}"
}
s3.getObject(req) { resp -> s3.getObject(req) { resp ->
ZipInputStream( ZipInputStream(
resp.body?.toInputStream() resp.body?.toInputStream()
?: throw IOException("S3 response is missing body") ?: throw IOException("S3 response is missing body"),
).use { zipStream -> ).use { zipStream ->
zipStream.decompress { name -> destination.resolve(name.takeLastWhile { it != '/' }) } zipStream.decompress(limit = 1) { name -> destination.resolve(name.takeLastWhile { it != '/' }) }
} }
} }
} }
@ -228,35 +255,39 @@ private fun UploadPartResponse.toCompletedPart(number: Int): CompletedPart {
* @return A ByteArray of the first `n` items. * @return A ByteArray of the first `n` items.
*/ */
private fun ByteArray.take(n: Int) = private fun ByteArray.take(n: Int) =
if (n == size) this // No copy if (n == size) {
else asList().subList(0, n).toByteArray() // TODO: One copy (toByteArray()), not sure how to do 0 copies here this // No copy
} else {
asList().subList(0, n).toByteArray() // TODO: One copy (toByteArray()), not sure how to do 0 copies here
}
/** /**
* Compress a file or directory as a ZIP file to an `OutputStream`. * Compress a file or directory as a ZIP file to an `OutputStream`.
* @param outputStream The `OutputStream` to write the ZIP file contents to. * @param outputStream The `OutputStream` to write the ZIP file contents to.
*/ */
private fun File.compressToZip(outputStream: OutputStream) = ZipOutputStream(outputStream).use { zipStream -> private fun File.compressToZip(outputStream: OutputStream) =
val parentDir = this.absoluteFile.parent + "/" ZipOutputStream(outputStream).use { zipStream ->
val fileQueue = ArrayDeque<File>() val parentDir = this.canonicalFile.parent + "/"
fileQueue.add(this) val fileQueue = ArrayDeque<File>()
fileQueue.forEach { subFile -> fileQueue.add(this)
val path = subFile.absolutePath.removePrefix(parentDir) fileQueue.forEach { subFile ->
val subFiles = subFile.listFiles() val path = subFile.canonicalPath.removePrefix(parentDir)
if (subFiles != null) { // Is a directory val subFiles = subFile.listFiles()
val entry = ZipEntry("$path/") if (subFiles != null) { // Is a directory
setZipAttributes(entry, subFile.toPath()) val entry = ZipEntry("$path/")
zipStream.putNextEntry(entry)
fileQueue.addAll(subFiles)
} else { // Otherwise, treat it as a file
BufferedInputStream(subFile.inputStream()).use { origin ->
val entry = ZipEntry(path)
setZipAttributes(entry, subFile.toPath()) setZipAttributes(entry, subFile.toPath())
zipStream.putNextEntry(entry) zipStream.putNextEntry(entry)
origin.copyTo(zipStream) fileQueue.addAll(subFiles)
} else { // Otherwise, treat it as a file
BufferedInputStream(subFile.inputStream()).use { origin ->
val entry = ZipEntry(path)
setZipAttributes(entry, subFile.toPath())
zipStream.putNextEntry(entry)
origin.copyTo(zipStream)
}
} }
} }
} }
}
/** /**
* Decompress `ZipInputStream` contents to specified destination paths. * Decompress `ZipInputStream` contents to specified destination paths.
@ -265,9 +296,11 @@ private fun File.compressToZip(outputStream: OutputStream) = ZipOutputStream(out
*/ */
private fun ZipInputStream.decompress( private fun ZipInputStream.decompress(
bufSize: Int = 1024 * 1024, bufSize: Int = 1024 * 1024,
entryNameToPath: (String) -> Path limit: Int? = null,
entryNameToPath: (String) -> Path,
) { ) {
var entry = this.nextEntry var entry = this.nextEntry
var count = 1
while (entry != null) { while (entry != null) {
val path = entryNameToPath(entry.name) val path = entryNameToPath(entry.name)
if (entry.isDirectory) { if (entry.isDirectory) {
@ -283,6 +316,9 @@ private fun ZipInputStream.decompress(
} }
} }
applyZipAttributes(entry, path) applyZipAttributes(entry, path)
// This is here, not in while loop, since we do not want to read more from the input stream.
// But this.nextEntry will read from the input stream.
if (limit != null && count++ >= limit) return
entry = this.nextEntry entry = this.nextEntry
} }
} }
@ -292,7 +328,10 @@ private fun ZipInputStream.decompress(
* @param entry The `ZipEntry` to set attributes of. * @param entry The `ZipEntry` to set attributes of.
* @param path The `Path` of the file to get the attributes from. * @param path The `Path` of the file to get the attributes from.
*/ */
private fun setZipAttributes(entry: ZipEntry, path: Path) { private fun setZipAttributes(
entry: ZipEntry,
path: Path,
) {
try { try {
val attrs = Files.getFileAttributeView(path, BasicFileAttributeView::class.java).readAttributes() val attrs = Files.getFileAttributeView(path, BasicFileAttributeView::class.java).readAttributes()
entry.setCreationTime(attrs.creationTime()) entry.setCreationTime(attrs.creationTime())
@ -307,7 +346,10 @@ private fun setZipAttributes(entry: ZipEntry, path: Path) {
* @param entry The `ZipEntry` to get the attributes from. * @param entry The `ZipEntry` to get the attributes from.
* @param path The `Path` of the file to set the attributes of. * @param path The `Path` of the file to set the attributes of.
*/ */
private fun applyZipAttributes(entry: ZipEntry, path: Path) { private fun applyZipAttributes(
entry: ZipEntry,
path: Path,
) {
try { try {
val attrs = Files.getFileAttributeView(path, BasicFileAttributeView::class.java) val attrs = Files.getFileAttributeView(path, BasicFileAttributeView::class.java)
attrs.setTimes(entry.lastModifiedTime, entry.lastAccessTime, entry.creationTime) attrs.setTimes(entry.lastModifiedTime, entry.lastAccessTime, entry.creationTime)

10
src/main/kotlin/backup/main.kt
View File

@ -1,10 +0,0 @@
package backup
import aws.sdk.kotlin.services.s3.S3Client
import kotlinx.coroutines.runBlocking
fun main() = runBlocking {
S3Client.fromEnvironment().use { s3 ->
val backupClient = BackupClient(s3, "teamcity-executors-test-task", 1024 * 1024 * 10)
}
}

104
src/main/kotlin/ziputils/CentralDirectoryFileHeader.kt
View File

@ -15,7 +15,7 @@ internal class CentralDirectoryFileHeader(
val disk: UShort, val disk: UShort,
val localHeaderOffset: UInt, val localHeaderOffset: UInt,
val fileName: String, val fileName: String,
val extraFieldRecords: List<ExtraFieldRecord> val extraFieldRecords: List<ExtraFieldRecord>,
) { ) {
val size: Int val size: Int
get() = SIZE + nameLength.toInt() + extraFieldLength.toInt() + commentLength.toInt() get() = SIZE + nameLength.toInt() + extraFieldLength.toInt() + commentLength.toInt()
@ -32,65 +32,85 @@ internal class CentralDirectoryFileHeader(
* @return A `CentralDirectoryFileHeader`. * @return A `CentralDirectoryFileHeader`.
*/ */
@Throws(InvalidDataException::class) @Throws(InvalidDataException::class)
fun fromByteArray(data: ByteArray, offset: Int): CentralDirectoryFileHeader { fun fromByteArray(
data: ByteArray,
offset: Int,
): CentralDirectoryFileHeader {
if (data.size - offset < SIZE) { if (data.size - offset < SIZE) {
throw InvalidDataException("CEN must be at least 46 bytes") throw InvalidDataException("CEN must be at least 46 bytes")
} }
val buf = ByteBuffer.wrap(data, offset, 46).order(ByteOrder.LITTLE_ENDIAN) val buf = ByteBuffer.wrap(data, offset, 46).order(ByteOrder.LITTLE_ENDIAN)
if (buf.getInt().toUInt() != SIGNATURE) { if (buf.getInt().toUInt() != SIGNATURE) {
throw InvalidSignatureException("Invalid signature") throw InvalidDataException("Invalid signature")
} }
val extraFieldRecords = mutableListOf<ExtraFieldRecord>() val extraFieldRecords = mutableListOf<ExtraFieldRecord>()
val nameLength = buf.getShort(offset + 28).toUShort() val nameLength = buf.getShort(offset + 28).toUShort()
buf.position(offset + 20) buf.position(offset + 20)
val cen = CentralDirectoryFileHeader( val cen =
compressedSize = buf.getInt().toUInt(), CentralDirectoryFileHeader(
uncompressedSize = buf.getInt().toUInt(), compressedSize = buf.getInt().toUInt(),
nameLength = nameLength uncompressedSize = buf.getInt().toUInt(),
.also { buf.position(offset + 30) }, nameLength =
extraFieldLength = buf.getShort().toUShort(), nameLength
commentLength = buf.getShort().toUShort(), .also { buf.position(offset + 30) },
disk = buf.getShort().toUShort() extraFieldLength = buf.getShort().toUShort(),
.also { buf.position(offset + 42) }, commentLength = buf.getShort().toUShort(),
localHeaderOffset = buf.getInt().toUInt(), disk =
fileName = String(data.sliceArray(offset + SIZE..<offset + SIZE + nameLength.toInt())), buf.getShort().toUShort()
extraFieldRecords = extraFieldRecords .also { buf.position(offset + 42) },
) localHeaderOffset = buf.getInt().toUInt(),
fileName = String(data.sliceArray(offset + SIZE..<offset + SIZE + nameLength.toInt())),
extraFieldRecords = extraFieldRecords,
)
if (data.size - offset < cen.size) { if (data.size - offset < cen.size) {
throw InvalidDataException("CEN is too short") throw InvalidDataException("CEN is too short")
} }
// Parse extra field records // Parse extra field records
val extraFieldsBuf = ByteBuffer.wrap( val extraFieldsBuf =
data, offset + SIZE + cen.nameLength.toInt(), cen.extraFieldLength.toInt() ByteBuffer.wrap(
).order(ByteOrder.LITTLE_ENDIAN) data,
offset + SIZE + cen.nameLength.toInt(),
cen.extraFieldLength.toInt(),
).order(ByteOrder.LITTLE_ENDIAN)
while (extraFieldsBuf.remaining() > 0) { while (extraFieldsBuf.remaining() > 0) {
val id = extraFieldsBuf.getShort().toUShort() val id = extraFieldsBuf.getShort().toUShort()
val size = extraFieldsBuf.getShort().toUShort() val size = extraFieldsBuf.getShort().toUShort()
extraFieldRecords.add(when (id) { extraFieldRecords.add(
Zip64ExtraFieldRecord.ID -> { when (id) {
Zip64ExtraFieldRecord( Zip64ExtraFieldRecord.ID -> {
size, Zip64ExtraFieldRecord(
if (cen.uncompressedSize == 0xffffffffU) { size,
extraFieldsBuf.getLong().toULong() if (cen.uncompressedSize == 0xffffffffU) {
} else null, extraFieldsBuf.getLong().toULong()
if (cen.compressedSize == 0xffffffffU) { } else {
extraFieldsBuf.getLong().toULong() null
} else null, },
if (cen.localHeaderOffset == 0xffffffffU) { if (cen.compressedSize == 0xffffffffU) {
extraFieldsBuf.getLong().toULong() extraFieldsBuf.getLong().toULong()
} else null, } else {
if (cen.disk == 0xffffU.toUShort()) { null
extraFieldsBuf.getInt().toUInt() },
} else null if (cen.localHeaderOffset == 0xffffffffU) {
) extraFieldsBuf.getLong().toULong()
} } else {
else -> { null
extraFieldsBuf.position(extraFieldsBuf.position() + size.toInt()) },
ExtraFieldRecord(id, size) if (cen.disk == 0xffffU.toUShort()) {
} extraFieldsBuf.getInt().toUInt()
}) } else {
null
},
)
}
else -> {
extraFieldsBuf.position(extraFieldsBuf.position() + size.toInt())
ExtraFieldRecord(id, size)
}
},
)
} }
return cen return cen

10
src/main/kotlin/ziputils/EndOfCentralDirectoryLocator.kt
View File

@ -7,11 +7,12 @@ import java.nio.ByteOrder
* Represents a partial ZIP64 end of central directory locator. * Represents a partial ZIP64 end of central directory locator.
*/ */
internal class EndOfCentralDirectoryLocator( internal class EndOfCentralDirectoryLocator(
val endOfCentralDirectory64Offset: ULong val endOfCentralDirectory64Offset: ULong,
) { ) {
companion object { companion object {
const val SIGNATURE = 0x07064b50U const val SIGNATURE = 0x07064b50U
const val SIZE = 20 const val SIZE = 20
/** /**
* Create `EndOfCentralDirectoryLocator` from raw byte data. * Create `EndOfCentralDirectoryLocator` from raw byte data.
* @throws InvalidDataException Provided `ByteArray` is not a supported EOCD locator. * @throws InvalidDataException Provided `ByteArray` is not a supported EOCD locator.
@ -20,13 +21,16 @@ internal class EndOfCentralDirectoryLocator(
* @return A `EndOfCentralDirectoryLocator`. * @return A `EndOfCentralDirectoryLocator`.
*/ */
@Throws(InvalidDataException::class) @Throws(InvalidDataException::class)
fun fromByteArray(data: ByteArray, offset: Int): EndOfCentralDirectoryLocator { fun fromByteArray(
data: ByteArray,
offset: Int,
): EndOfCentralDirectoryLocator {
if (data.size - offset < SIZE) { if (data.size - offset < SIZE) {
throw InvalidDataException("EOCD64 locator must be at least 20 bytes") throw InvalidDataException("EOCD64 locator must be at least 20 bytes")
} }
val buf = ByteBuffer.wrap(data, offset, SIZE).order(ByteOrder.LITTLE_ENDIAN) val buf = ByteBuffer.wrap(data, offset, SIZE).order(ByteOrder.LITTLE_ENDIAN)
if (buf.getInt().toUInt() != SIGNATURE) { if (buf.getInt().toUInt() != SIGNATURE) {
throw InvalidSignatureException("Invalid signature") throw InvalidDataException("Invalid signature")
} }
buf.position(offset + 8) buf.position(offset + 8)
return EndOfCentralDirectoryLocator(buf.getLong().toULong()) return EndOfCentralDirectoryLocator(buf.getLong().toULong())

15
src/main/kotlin/ziputils/EndOfCentralDirectoryRecord.kt
View File

@ -7,14 +7,14 @@ import java.nio.ByteOrder
* Represents a partial ZIP end of central directory record. * Represents a partial ZIP end of central directory record.
*/ */
internal class EndOfCentralDirectoryRecord( internal class EndOfCentralDirectoryRecord(
val centralDirectoryOffset: UInt val centralDirectoryOffset: UInt,
) { ) {
fun eocd64Required(): Boolean = fun eocd64Required(): Boolean = centralDirectoryOffset == 0xffffffffU
centralDirectoryOffset == 0xffffffffU
companion object { companion object {
const val SIGNATURE = 0x06054b50U const val SIGNATURE = 0x06054b50U
const val SIZE = 22 const val SIZE = 22
/** /**
* Create `EndOfCentralDirectoryRecord` from raw byte data. * Create `EndOfCentralDirectoryRecord` from raw byte data.
* @throws InvalidDataException Provided `ByteArray` is not a supported EOCD64. * @throws InvalidDataException Provided `ByteArray` is not a supported EOCD64.
@ -23,17 +23,20 @@ internal class EndOfCentralDirectoryRecord(
* @return A `EndOfCentralDirectoryRecord`. * @return A `EndOfCentralDirectoryRecord`.
*/ */
@Throws(InvalidDataException::class) @Throws(InvalidDataException::class)
fun fromByteArray(data: ByteArray, offset: Int): EndOfCentralDirectoryRecord { fun fromByteArray(
data: ByteArray,
offset: Int,
): EndOfCentralDirectoryRecord {
if (data.size - offset < SIZE) { if (data.size - offset < SIZE) {
throw InvalidDataException("EOCD must be at least 22 bytes") throw InvalidDataException("EOCD must be at least 22 bytes")
} }
val buf = ByteBuffer.wrap(data, offset, SIZE).order(ByteOrder.LITTLE_ENDIAN) val buf = ByteBuffer.wrap(data, offset, SIZE).order(ByteOrder.LITTLE_ENDIAN)
if (buf.getInt().toUInt() != SIGNATURE) { if (buf.getInt().toUInt() != SIGNATURE) {
throw InvalidSignatureException("Invalid signature") throw InvalidDataException("Invalid signature")
} }
buf.position(offset + 16) buf.position(offset + 16)
return EndOfCentralDirectoryRecord( return EndOfCentralDirectoryRecord(
centralDirectoryOffset = buf.getInt().toUInt() centralDirectoryOffset = buf.getInt().toUInt(),
) )
} }
} }

12
src/main/kotlin/ziputils/EndOfCentralDirectoryRecord64.kt
View File

@ -7,11 +7,12 @@ import java.nio.ByteOrder
* Represents a partial ZIP64 end of central directory record. * Represents a partial ZIP64 end of central directory record.
*/ */
internal class EndOfCentralDirectoryRecord64( internal class EndOfCentralDirectoryRecord64(
val centralDirectoryOffset: ULong val centralDirectoryOffset: ULong,
) { ) {
companion object { companion object {
const val SIGNATURE = 0x06064b50U const val SIGNATURE = 0x06064b50U
const val SIZE = 56 const val SIZE = 56
/** /**
* Create `EndOfCentralDirectoryRecord64` from raw byte data. * Create `EndOfCentralDirectoryRecord64` from raw byte data.
* @throws InvalidDataException Provided `ByteArray` is not a supported EOCD. * @throws InvalidDataException Provided `ByteArray` is not a supported EOCD.
@ -20,17 +21,20 @@ internal class EndOfCentralDirectoryRecord64(
* @return A `EndOfCentralDirectoryRecord64`. * @return A `EndOfCentralDirectoryRecord64`.
*/ */
@Throws(InvalidDataException::class) @Throws(InvalidDataException::class)
fun fromByteArray(data: ByteArray, offset: Int): EndOfCentralDirectoryRecord64 { fun fromByteArray(
data: ByteArray,
offset: Int,
): EndOfCentralDirectoryRecord64 {
if (data.size - offset < SIZE) { if (data.size - offset < SIZE) {
throw InvalidDataException("EOCD64 must be at least 56 bytes") throw InvalidDataException("EOCD64 must be at least 56 bytes")
} }
val buf = ByteBuffer.wrap(data, offset, SIZE).order(ByteOrder.LITTLE_ENDIAN) val buf = ByteBuffer.wrap(data, offset, SIZE).order(ByteOrder.LITTLE_ENDIAN)
if (buf.getInt().toUInt() != SIGNATURE) { if (buf.getInt().toUInt() != SIGNATURE) {
throw InvalidSignatureException("Invalid signature") throw InvalidDataException("Invalid signature")
} }
buf.position(offset + 48) buf.position(offset + 48)
return EndOfCentralDirectoryRecord64( return EndOfCentralDirectoryRecord64(
centralDirectoryOffset = buf.getLong().toULong() centralDirectoryOffset = buf.getLong().toULong(),
) )
} }
} }

11
src/main/kotlin/ziputils/Exceptions.kt
View File

@ -1,11 +0,0 @@
package ziputils
/**
* Represents an invalid raw byte data exception.
*/
class InvalidDataException(message: String): Exception(message)
/**
* Represents an invalid raw byte signature exception.
*/
class InvalidSignatureException(message: String): Exception(message)

2
src/main/kotlin/ziputils/ExtraFieldRecord.kt
View File

@ -5,5 +5,5 @@ package ziputils
*/ */
internal open class ExtraFieldRecord( internal open class ExtraFieldRecord(
val id: UShort, val id: UShort,
val size: UShort val size: UShort,
) )

6
src/main/kotlin/ziputils/InvalidDataException.kt Normal file
View File

@ -0,0 +1,6 @@
package ziputils
/**
* Represents an invalid raw byte data exception.
*/
class InvalidDataException(message: String) : Exception(message)

4
src/main/kotlin/ziputils/Zip64ExtraFieldRecord.kt
View File

@ -8,8 +8,8 @@ internal class Zip64ExtraFieldRecord(
val uncompressedSize: ULong?, val uncompressedSize: ULong?,
val compressedSize: ULong?, val compressedSize: ULong?,
val localHeaderOffset: ULong?, val localHeaderOffset: ULong?,
val disk: UInt? val disk: UInt?,
): ExtraFieldRecord(ID, size) { ) : ExtraFieldRecord(ID, size) {
companion object { companion object {
const val ID: UShort = 0x0001U const val ID: UShort = 0x0001U
} }