bulk-cdk-toolkit-load-object-storage/io.airbyte.cdk.load.file.object_storage

Package-level declarations

Types

class AvroFormattingWriter(stream: DestinationStream, outputStream: OutputStream, formatConfig: AvroFormatConfiguration, rootLevelFlattening: Boolean) : ObjectStorageFormattingWriter

BufferedFormattingWriter

class BufferedFormattingWriter<T : OutputStream>(writer: ObjectStorageFormattingWriter, buffer: ManagedByteArrayOutputStream, streamProcessor: StreamProcessor<T>, wrappingBuffer: T) : ObjectStorageFormattingWriter

BufferedFormattingWriterFactory

@Singleton

@Secondary

class BufferedFormattingWriterFactory<T : OutputStream>(writerFactory: ObjectStorageFormattingWriterFactory, compressionConfigurationProvider: ObjectStorageCompressionConfigurationProvider<T>, @Named(value = "dataChannelMedium") dataChannelMedium: DataChannelMedium)

ByteArrayPool

object ByteArrayPool

Re-uses large byte arrays to cut GC pressure when we generate many S3 parts. Keep at most 512 MiB of slabs in the pool.

CSVFormattingWriter

class CSVFormattingWriter(stream: DestinationStream, outputStream: OutputStream, rootLevelFlattening: Boolean, extractedAtAsTimestampWithTimezone: Boolean) : ObjectStorageFormattingWriter

DefaultObjectStorageFormattingWriterFactory

@Singleton

@Secondary

class DefaultObjectStorageFormattingWriterFactory(formatConfigProvider: ObjectStorageFormatConfigurationProvider, @Named(value = "dataChannelFormat") dataChannelFormat: DataChannelFormat) : ObjectStorageFormattingWriterFactory

JsonFormattingWriter

class JsonFormattingWriter(stream: DestinationStream, outputStream: OutputStream, rootLevelFlattening: Boolean) : ObjectStorageFormattingWriter

ManagedByteArrayOutputStream

abstract class ManagedByteArrayOutputStream(initialCapacity: Int) : ByteArrayOutputStream

ObjectStorageClient

interface ObjectStorageClient<T : RemoteObject<*>>

ObjectStorageFormattingWriter

interface ObjectStorageFormattingWriter : Closeable

ObjectStorageFormattingWriterFactory

interface ObjectStorageFormattingWriterFactory

ObjectStoragePathFactory

@Singleton

@Secondary

class ObjectStoragePathFactory(pathConfigProvider: ObjectStoragePathConfigurationProvider, formatConfigProvider: ObjectStorageFormatConfigurationProvider? = null, compressionConfigProvider: ObjectStorageCompressionConfigurationProvider<*>? = null, timeProvider: TimeProvider) : PathFactory

ParquetFormattingWriter

class ParquetFormattingWriter(stream: DestinationStream, outputStream: OutputStream, formatConfig: ParquetFormatConfiguration, rootLevelFlattening: Boolean) : ObjectStorageFormattingWriter

Part

data class Part(val key: String, val fileNumber: Long, val partIndex: Int, val bytes: ByteArray?, val isFinal: Boolean)

Reassembles part metadata into a view of the upload state.

PartBookkeeper

class PartBookkeeper

PartFactory

class PartFactory(val key: String, val fileNumber: Long)

Generates part w/ metadata for a multi-part upload for a given key and file no. parts are 1-indexed. For convenience, empty parts are tolerated but not counted by the assembler.

PathFactory

interface PathFactory

PathMatcher

data class PathMatcher(val regex: Regex, val variableToIndex: Map<String, Int>)

PathMatcherException

class PathMatcherException(val message: String, val cause: Throwable) : Throwable

PathMatcherResult

data class PathMatcherResult(val path: String, val partNumber: Long?, val customSuffix: String?)

PooledByteArrayOutputStream

class PooledByteArrayOutputStream(minCapacity: Int = 32 * 1024) : ManagedByteArrayOutputStream

ProtoToAvroFormatter

class ProtoToAvroFormatter(stream: DestinationStream, outputStream: OutputStream, formatConfig: AvroFormatConfiguration, rootLevelFlattening: Boolean) : ObjectStorageFormattingWriter

ProtoToCsvFormatter

class ProtoToCsvFormatter(stream: DestinationStream, outputStream: OutputStream, rootLevelFlattening: Boolean, extractedAtAsTimestampWithTimezone: Boolean) : ObjectStorageFormattingWriter

ProtoToCsvWriter

class ProtoToCsvWriter(header: Array<String>, columns: Array<AirbyteValueProxy.FieldAccessor>, rootLevelFlattening: Boolean, extractedAtTsTz: Boolean)

ProtoToJsonFormatter

class ProtoToJsonFormatter(stream: DestinationStream, outputStream: OutputStream, rootLevelFlattening: Boolean) : ObjectStorageFormattingWriter

ProtoToJsonWriter

class ProtoToJsonWriter(columns: Array<AirbyteValueProxy.FieldAccessor>, flatten: Boolean)

ProtoToParquetFormatter

class ProtoToParquetFormatter(stream: DestinationStream, outputStream: OutputStream, formatConfig: ParquetFormatConfiguration, rootLevelFlattening: Boolean) : ObjectStorageFormattingWriter

RemoteObject

interface RemoteObject<C>

StandardByteArrayOutputStream

class StandardByteArrayOutputStream : ManagedByteArrayOutputStream

StreamingUpload

interface StreamingUpload<T : RemoteObject<*>>