bulk-cdk-toolkit-load-object-storage/io.airbyte.cdk.load.pipline.object_storage.file

Package-level declarations

Types

@Singleton

@Requires(condition = IsFileTransferCondition::class)

class FileChunkStep<T : RemoteObject<*>>(catalog: DestinationCatalog, fileLoader: ObjectLoader, @Named(value = "fileQueue") inputQueue: PartitionedQueue<PipelineEvent<StreamKey, DestinationRecordRaw>>, @Named(value = "filePartQueue") partQueue: PartitionedQueue<PipelineEvent<ObjectKey, ObjectLoaderPartFormatter.FormattedPart>>, pathFactory: ObjectStoragePathFactory, uploadIdGenerator: UploadIdGenerator) : LoadPipelineStep

FileChunkTask

class FileChunkTask<T>(loader: ObjectLoader, catalog: DestinationCatalog, pathFactory: ObjectStoragePathFactory, fileHandleFactory: FileHandleFactory, uploadIdGenerator: UploadIdGenerator, inputQueue: PartitionedQueue<PipelineEvent<StreamKey, DestinationRecordRaw>>, partQueue: PartitionedQueue<PipelineEvent<ObjectKey, ObjectLoaderPartFormatter.FormattedPart>>, partitioner: ObjectLoaderFormattedPartPartitioner<StreamKey, T>, partition: Int) : Task

Given an input stream of file references, reads the files and chunks them into parts, emitting those parts to the partQueue. Once finished chunking a file, it deletes it from local storage.

FileCompletedOutputPartitioner

@Singleton

@Named(value = "fileCompletedOutputPartitioner")

class FileCompletedOutputPartitioner<K : WithStream, T : RemoteObject<*>> : ObjectLoaderCompletedUploadPartitioner<K, ObjectLoaderPartLoader.PartResult<T>, StreamKey, T>

FileHandle

class FileHandle(pathName: String)

File in Kotlin has built-n extensions cannot be directly mocked. This wrapper allows us to mock.

FileHandleFactory

class FileHandleFactory

Exists to make it easier to test file access (read, delete, etc.) by injecting a mock factory.

FilePartIterator

class FilePartIterator(inputStream: InputStream, partSizeBytes: Int, finalPath: String) : Iterator<Part>

Turns a file into a series of Part chunks.

ForwardFileRecordStep

@Singleton

@Requires(condition = IsFileTransferCondition::class)

class ForwardFileRecordStep<T>(@Named(value = "fileCompletedQueue") inputQueue: PartitionedQueue<PipelineEvent<StreamKey, ObjectLoaderUploadCompleter.UploadResult<T>>>, @Named(value = "recordQueue") outputQueue: PartitionedQueue<PipelineEvent<StreamKey, DestinationRecordRaw>>) : LoadPipelineStep

ForwardFileRecordTask

class ForwardFileRecordTask<T>(inputQueue: PartitionedQueue<PipelineEvent<StreamKey, ObjectLoaderUploadCompleter.UploadResult<T>>>, outputQueue: PartitionedQueue<PipelineEvent<StreamKey, DestinationRecordRaw>>, partition: Int) : Task

ProcessFileTaskLegacy

@Singleton

class ProcessFileTaskLegacy(@Named(value = "fileMessageQueue") inputQueue: MessageQueue<FileTransferQueueMessage>, @Named(value = "objectLoaderPartQueue") outputQueue: PartitionedQueue<PipelineEvent<ObjectKey, ObjectLoaderPartFormatter.FormattedPart>>, partAccumulatorFactory: FilePartAccumulatorFactory) : Task

ProcessFileTaskLegacyStep

@Singleton

class ProcessFileTaskLegacyStep(processFileTaskLegacy: ProcessFileTaskLegacy) : LoadPipelineStep

RouteEventStep

@Singleton

@Requires(condition = IsFileTransferCondition::class)

class RouteEventStep(catalog: DestinationCatalog, @Named(value = "dataChannelInputFlows") inputFlows: Array<Flow<PipelineEvent<StreamKey, DestinationRecordRaw>>>, @Named(value = "fileQueue") fileQueue: PartitionedQueue<PipelineEvent<StreamKey, DestinationRecordRaw>>, @Named(value = "recordQueue") recordQueue: PartitionedQueue<PipelineEvent<StreamKey, DestinationRecordRaw>>) : LoadPipelineStep

RouteEventTask

class RouteEventTask(catalog: DestinationCatalog, inputFlow: Flow<PipelineEvent<StreamKey, DestinationRecordRaw>>, fileQueue: PartitionedQueue<PipelineEvent<StreamKey, DestinationRecordRaw>>, recordQueue: PartitionedQueue<PipelineEvent<StreamKey, DestinationRecordRaw>>, partition: Int) : Task

UploadIdGenerator

@Singleton

class UploadIdGenerator(uuidGenerator: UUIDGenerator)

Generate a unique upload id to keep track of the upload in the case of file name collisions. Factored out for testability.