Compare commits
No commits in common. "b8a920d0e7e556303525bd6b671378936af9544d" and "c802c76ace2da526c8149f27878b2dd15a365e56" have entirely different histories.
b8a920d0e7
...
c802c76ace
|
@ -41,7 +41,7 @@ jobs:
|
||||||
run: ./gradlew shadowJar
|
run: ./gradlew shadowJar
|
||||||
|
|
||||||
- name: Upload artifacts
|
- name: Upload artifacts
|
||||||
uses: actions/upload-artifact@v3
|
uses: actions/upload-artifact@v4
|
||||||
with:
|
with:
|
||||||
name: ShadowJAR
|
name: ShadowJAR
|
||||||
path: build/libs/*-all.jar
|
path: build/libs/*-all.jar
|
|
@ -24,18 +24,11 @@ import java.util.zip.ZipInputStream
|
||||||
import java.util.zip.ZipOutputStream
|
import java.util.zip.ZipOutputStream
|
||||||
import kotlin.io.path.createDirectory
|
import kotlin.io.path.createDirectory
|
||||||
|
|
||||||
/**
|
|
||||||
* AWS S3 backup client.
|
|
||||||
*/
|
|
||||||
class BackupClient(
|
class BackupClient(
|
||||||
private val s3: S3Client,
|
private val s3: S3Client,
|
||||||
private val bucketName: String,
|
private val bucketName: String,
|
||||||
private val bufSize: Int = 1024 * 1024 * 100
|
private val bufSize: Int = 1024 * 1024 * 100
|
||||||
) {
|
) {
|
||||||
/**
|
|
||||||
* Upload a file/directory backup to AWS S3.
|
|
||||||
* @param file The File object for the file or directory.
|
|
||||||
*/
|
|
||||||
suspend fun upload(file: File) = coroutineScope {
|
suspend fun upload(file: File) = coroutineScope {
|
||||||
val backupKey = "${file.name}/${Instant.now()}.zip"
|
val backupKey = "${file.name}/${Instant.now()}.zip"
|
||||||
PipedInputStream().use { inputStream ->
|
PipedInputStream().use { inputStream ->
|
||||||
|
@ -65,7 +58,7 @@ class BackupClient(
|
||||||
partNumber = number
|
partNumber = number
|
||||||
uploadId = upload.uploadId
|
uploadId = upload.uploadId
|
||||||
body = ByteStream.fromBytes(data.take(bytesRead))
|
body = ByteStream.fromBytes(data.take(bytesRead))
|
||||||
}.toCompletedPart(number)
|
}.asCompletedPart(number)
|
||||||
uploadParts.add(part)
|
uploadParts.add(part)
|
||||||
number++
|
number++
|
||||||
bytesRead = inputStream.readNBytes(data, 0, bufSize)
|
bytesRead = inputStream.readNBytes(data, 0, bufSize)
|
||||||
|
@ -99,11 +92,6 @@ class BackupClient(
|
||||||
backupKey
|
backupKey
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Restore a backup from AWS S3.
|
|
||||||
* @param destination The destination directory path for the backup contents.
|
|
||||||
* @param backupKey The S3 key of the backup.
|
|
||||||
*/
|
|
||||||
suspend fun restore(destination: Path, backupKey: String) = coroutineScope {
|
suspend fun restore(destination: Path, backupKey: String) = coroutineScope {
|
||||||
val req = GetObjectRequest {
|
val req = GetObjectRequest {
|
||||||
bucket = bucketName
|
bucket = bucketName
|
||||||
|
@ -119,12 +107,6 @@ class BackupClient(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Restore a single file from a backup from AWS S3.
|
|
||||||
* @param destination The destination directory path for the file from the backup.
|
|
||||||
* @param backupKey The S3 key of the backup.
|
|
||||||
* @param fileName The full name of the file to restore (including directories if it was under a subdirectory).
|
|
||||||
*/
|
|
||||||
suspend fun restoreFile(destination: Path, backupKey: String, fileName: String) = coroutineScope {
|
suspend fun restoreFile(destination: Path, backupKey: String, fileName: String) = coroutineScope {
|
||||||
// For byte ranges refer to https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.9.TXT
|
// For byte ranges refer to https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.9.TXT
|
||||||
val eocdReq = GetObjectRequest {
|
val eocdReq = GetObjectRequest {
|
||||||
|
@ -205,12 +187,7 @@ class BackupClient(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
private fun UploadPartResponse.asCompletedPart(number: Int): CompletedPart {
|
||||||
* Convert an UploadPartResponse to a CompletedPart.
|
|
||||||
* @param number The part number that was used for this part upload.
|
|
||||||
* @return The CompletedPart object.
|
|
||||||
*/
|
|
||||||
private fun UploadPartResponse.toCompletedPart(number: Int): CompletedPart {
|
|
||||||
val part = this
|
val part = this
|
||||||
return CompletedPart {
|
return CompletedPart {
|
||||||
partNumber = number
|
partNumber = number
|
||||||
|
@ -222,19 +199,10 @@ private fun UploadPartResponse.toCompletedPart(number: Int): CompletedPart {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Take first `n` items from the beginning of a ByteArray.
|
|
||||||
* @param n The number of items to take.
|
|
||||||
* @return A ByteArray of the first `n` items.
|
|
||||||
*/
|
|
||||||
private fun ByteArray.take(n: Int) =
|
private fun ByteArray.take(n: Int) =
|
||||||
if (n == size) this // No copy
|
if (n == size) this // No copy
|
||||||
else asList().subList(0, n).toByteArray() // TODO: One copy (toByteArray()), not sure how to do 0 copies here
|
else asList().subList(0, n).toByteArray() // TODO: One copy (toByteArray()), not sure how to do 0 copies here
|
||||||
|
|
||||||
/**
|
|
||||||
* Compress a file or directory as a ZIP file to an `OutputStream`.
|
|
||||||
* @param outputStream The `OutputStream` to write the ZIP file contents to.
|
|
||||||
*/
|
|
||||||
private fun File.compressToZip(outputStream: OutputStream) = ZipOutputStream(outputStream).use { zipStream ->
|
private fun File.compressToZip(outputStream: OutputStream) = ZipOutputStream(outputStream).use { zipStream ->
|
||||||
val parentDir = this.absoluteFile.parent + "/"
|
val parentDir = this.absoluteFile.parent + "/"
|
||||||
val fileQueue = ArrayDeque<File>()
|
val fileQueue = ArrayDeque<File>()
|
||||||
|
@ -258,11 +226,6 @@ private fun File.compressToZip(outputStream: OutputStream) = ZipOutputStream(out
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Decompress `ZipInputStream` contents to specified destination paths.
|
|
||||||
* @param bufSize The buffer size to use for writing the decompressed files.
|
|
||||||
* @param entryNameToPath A function to convert ZIP entry names to destination `Path`s.
|
|
||||||
*/
|
|
||||||
private fun ZipInputStream.decompress(
|
private fun ZipInputStream.decompress(
|
||||||
bufSize: Int = 1024 * 1024,
|
bufSize: Int = 1024 * 1024,
|
||||||
entryNameToPath: (String) -> Path
|
entryNameToPath: (String) -> Path
|
||||||
|
@ -287,11 +250,6 @@ private fun ZipInputStream.decompress(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Set a `ZipEntry`'s attributes given a file's path.
|
|
||||||
* @param entry The `ZipEntry` to set attributes of.
|
|
||||||
* @param path The `Path` of the file to get the attributes from.
|
|
||||||
*/
|
|
||||||
private fun setZipAttributes(entry: ZipEntry, path: Path) {
|
private fun setZipAttributes(entry: ZipEntry, path: Path) {
|
||||||
try {
|
try {
|
||||||
val attrs = Files.getFileAttributeView(path, BasicFileAttributeView::class.java).readAttributes()
|
val attrs = Files.getFileAttributeView(path, BasicFileAttributeView::class.java).readAttributes()
|
||||||
|
@ -302,11 +260,6 @@ private fun setZipAttributes(entry: ZipEntry, path: Path) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Set a file's attributes given a `ZipEntry`.
|
|
||||||
* @param entry The `ZipEntry` to get the attributes from.
|
|
||||||
* @param path The `Path` of the file to set the attributes of.
|
|
||||||
*/
|
|
||||||
private fun applyZipAttributes(entry: ZipEntry, path: Path) {
|
private fun applyZipAttributes(entry: ZipEntry, path: Path) {
|
||||||
try {
|
try {
|
||||||
val attrs = Files.getFileAttributeView(path, BasicFileAttributeView::class.java)
|
val attrs = Files.getFileAttributeView(path, BasicFileAttributeView::class.java)
|
||||||
|
|
|
@ -1,7 +1,11 @@
|
||||||
package backup
|
package backup
|
||||||
|
|
||||||
import aws.sdk.kotlin.services.s3.S3Client
|
import aws.sdk.kotlin.services.s3.S3Client
|
||||||
|
import aws.sdk.kotlin.services.s3.model.ListBucketsRequest
|
||||||
|
import kotlinx.coroutines.Dispatchers
|
||||||
import kotlinx.coroutines.runBlocking
|
import kotlinx.coroutines.runBlocking
|
||||||
|
import java.io.File
|
||||||
|
import kotlin.io.path.Path
|
||||||
|
|
||||||
fun main() = runBlocking {
|
fun main() = runBlocking {
|
||||||
S3Client.fromEnvironment().use { s3 ->
|
S3Client.fromEnvironment().use { s3 ->
|
||||||
|
|
|
@ -3,9 +3,6 @@ package ziputils
|
||||||
import java.nio.ByteBuffer
|
import java.nio.ByteBuffer
|
||||||
import java.nio.ByteOrder
|
import java.nio.ByteOrder
|
||||||
|
|
||||||
/**
|
|
||||||
* Represents a partial ZIP central directory file header.
|
|
||||||
*/
|
|
||||||
internal class CentralDirectoryFileHeader(
|
internal class CentralDirectoryFileHeader(
|
||||||
val compressedSize: UInt,
|
val compressedSize: UInt,
|
||||||
val uncompressedSize: UInt,
|
val uncompressedSize: UInt,
|
||||||
|
@ -25,11 +22,8 @@ internal class CentralDirectoryFileHeader(
|
||||||
const val SIZE = 46
|
const val SIZE = 46
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create `CentralDirectoryFileHeader` from raw byte data.
|
* Create CentralDirectoryFileHeader from raw byte data.
|
||||||
* @throws InvalidDataException provided `ByteArray` is not a supported CEN.
|
* @throws InvalidDataException provided ByteArray is not a supported CEN.
|
||||||
* @param data Raw byte data.
|
|
||||||
* @param offset Skip first <offset> bytes in data array.
|
|
||||||
* @return A `CentralDirectoryFileHeader`.
|
|
||||||
*/
|
*/
|
||||||
@Throws(InvalidDataException::class)
|
@Throws(InvalidDataException::class)
|
||||||
fun fromByteArray(data: ByteArray, offset: Int): CentralDirectoryFileHeader {
|
fun fromByteArray(data: ByteArray, offset: Int): CentralDirectoryFileHeader {
|
||||||
|
|
|
@ -3,22 +3,12 @@ package ziputils
|
||||||
import java.nio.ByteBuffer
|
import java.nio.ByteBuffer
|
||||||
import java.nio.ByteOrder
|
import java.nio.ByteOrder
|
||||||
|
|
||||||
/**
|
|
||||||
* Represents a partial ZIP64 end of central directory locator.
|
|
||||||
*/
|
|
||||||
internal class EndOfCentralDirectoryLocator(
|
internal class EndOfCentralDirectoryLocator(
|
||||||
val endOfCentralDirectory64Offset: ULong
|
val endOfCentralDirectory64Offset: ULong
|
||||||
) {
|
) {
|
||||||
companion object {
|
companion object {
|
||||||
const val SIGNATURE = 0x07064b50U
|
const val SIGNATURE = 0x07064b50U
|
||||||
const val SIZE = 20
|
const val SIZE = 20
|
||||||
/**
|
|
||||||
* Create `EndOfCentralDirectoryLocator` from raw byte data.
|
|
||||||
* @throws InvalidDataException Provided `ByteArray` is not a supported EOCD locator.
|
|
||||||
* @param data Raw byte data.
|
|
||||||
* @param offset Skip first <offset> bytes in data array.
|
|
||||||
* @return A `EndOfCentralDirectoryLocator`.
|
|
||||||
*/
|
|
||||||
@Throws(InvalidDataException::class)
|
@Throws(InvalidDataException::class)
|
||||||
fun fromByteArray(data: ByteArray, offset: Int): EndOfCentralDirectoryLocator {
|
fun fromByteArray(data: ByteArray, offset: Int): EndOfCentralDirectoryLocator {
|
||||||
if (data.size - offset < SIZE) {
|
if (data.size - offset < SIZE) {
|
||||||
|
|
|
@ -4,7 +4,8 @@ import java.nio.ByteBuffer
|
||||||
import java.nio.ByteOrder
|
import java.nio.ByteOrder
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Represents a partial ZIP end of central directory record.
|
* Partial End of Central Directory record class.
|
||||||
|
* Only supports data required by the backup tool.
|
||||||
*/
|
*/
|
||||||
internal class EndOfCentralDirectoryRecord(
|
internal class EndOfCentralDirectoryRecord(
|
||||||
val centralDirectoryOffset: UInt
|
val centralDirectoryOffset: UInt
|
||||||
|
@ -16,11 +17,8 @@ internal class EndOfCentralDirectoryRecord(
|
||||||
const val SIGNATURE = 0x06054b50U
|
const val SIGNATURE = 0x06054b50U
|
||||||
const val SIZE = 22
|
const val SIZE = 22
|
||||||
/**
|
/**
|
||||||
* Create `EndOfCentralDirectoryRecord` from raw byte data.
|
* Create EndOfCentralDirectoryRecord from raw byte data.
|
||||||
* @throws InvalidDataException Provided `ByteArray` is not a supported EOCD64.
|
* @throws InvalidDataException provided ByteArray is not a supported EOCD64.
|
||||||
* @param data Raw byte data.
|
|
||||||
* @param offset Skip first <offset> bytes in data array.
|
|
||||||
* @return A `EndOfCentralDirectoryRecord`.
|
|
||||||
*/
|
*/
|
||||||
@Throws(InvalidDataException::class)
|
@Throws(InvalidDataException::class)
|
||||||
fun fromByteArray(data: ByteArray, offset: Int): EndOfCentralDirectoryRecord {
|
fun fromByteArray(data: ByteArray, offset: Int): EndOfCentralDirectoryRecord {
|
||||||
|
|
|
@ -4,7 +4,8 @@ import java.nio.ByteBuffer
|
||||||
import java.nio.ByteOrder
|
import java.nio.ByteOrder
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Represents a partial ZIP64 end of central directory record.
|
* Partial End of Central Directory record (ZIP64) class.
|
||||||
|
* Only supports data required by the backup tool.
|
||||||
*/
|
*/
|
||||||
internal class EndOfCentralDirectoryRecord64(
|
internal class EndOfCentralDirectoryRecord64(
|
||||||
val centralDirectoryOffset: ULong
|
val centralDirectoryOffset: ULong
|
||||||
|
@ -13,11 +14,8 @@ internal class EndOfCentralDirectoryRecord64(
|
||||||
const val SIGNATURE = 0x06064b50U
|
const val SIGNATURE = 0x06064b50U
|
||||||
const val SIZE = 56
|
const val SIZE = 56
|
||||||
/**
|
/**
|
||||||
* Create `EndOfCentralDirectoryRecord64` from raw byte data.
|
* Create EndOfCentralDirectoryRecord64 from raw byte data.
|
||||||
* @throws InvalidDataException Provided `ByteArray` is not a supported EOCD.
|
* @throws InvalidDataException provided ByteArray is not a supported EOCD.
|
||||||
* @param data Raw byte data.
|
|
||||||
* @param offset Skip first <offset> bytes in data array.
|
|
||||||
* @return A `EndOfCentralDirectoryRecord64`.
|
|
||||||
*/
|
*/
|
||||||
@Throws(InvalidDataException::class)
|
@Throws(InvalidDataException::class)
|
||||||
fun fromByteArray(data: ByteArray, offset: Int): EndOfCentralDirectoryRecord64 {
|
fun fromByteArray(data: ByteArray, offset: Int): EndOfCentralDirectoryRecord64 {
|
||||||
|
|
|
@ -1,11 +1,4 @@
|
||||||
package ziputils
|
package ziputils
|
||||||
|
|
||||||
/**
|
|
||||||
* Represents an invalid raw byte data exception.
|
|
||||||
*/
|
|
||||||
class InvalidDataException(message: String): Exception(message)
|
class InvalidDataException(message: String): Exception(message)
|
||||||
|
|
||||||
/**
|
|
||||||
* Represents an invalid raw byte signature exception.
|
|
||||||
*/
|
|
||||||
class InvalidSignatureException(message: String): Exception(message)
|
class InvalidSignatureException(message: String): Exception(message)
|
||||||
|
|
|
@ -1,8 +1,5 @@
|
||||||
package ziputils
|
package ziputils
|
||||||
|
|
||||||
/**
|
|
||||||
* Represents a partial ZIP extra field record.
|
|
||||||
*/
|
|
||||||
internal open class ExtraFieldRecord(
|
internal open class ExtraFieldRecord(
|
||||||
val id: UShort,
|
val id: UShort,
|
||||||
val size: UShort
|
val size: UShort
|
||||||
|
|
|
@ -1,8 +1,5 @@
|
||||||
package ziputils
|
package ziputils
|
||||||
|
|
||||||
/**
|
|
||||||
* Represents a ZIP ZIP64 extra field record (ID 0x0001).
|
|
||||||
*/
|
|
||||||
internal class Zip64ExtraFieldRecord(
|
internal class Zip64ExtraFieldRecord(
|
||||||
size: UShort,
|
size: UShort,
|
||||||
val uncompressedSize: ULong?,
|
val uncompressedSize: ULong?,
|
||||||
|
|
Reference in New Issue