bulk-cdk-toolkit-load-object-storage/io.airbyte.cdk.load.factory.object_storage/ObjectLoaderStepBeanFactory

ObjectLoaderStepBeanFactory

@Factory
class ObjectLoaderStepBeanFactory

Constructors

Link copied to clipboard
constructor()

Functions

Link copied to clipboard
@Named(value = "filePartLoaderStep")
@Singleton
@Requires(condition = IsFileTransferCondition::class)
fun <T : RemoteObject<*>> filePartLoader(loader: ObjectLoader, partLoader: ObjectLoaderPartLoader<T>, @Named(value = "filePartQueue") inputQueue: PartitionedQueue<PipelineEvent<ObjectKey, ObjectLoaderPartFormatter.FormattedPart>>, @Named(value = "fileLoadedPartQueue") outputQueue: PartitionedQueue<PipelineEvent<ObjectKey, ObjectLoaderPartLoader.PartResult<T>>>, taskFactory: LoadPipelineStepTaskFactory): ObjectLoaderPartLoaderStep<T>
Link copied to clipboard
@Named(value = "fileRecordPartFormatterStep")
@Singleton
@Requires(condition = IsFileTransferCondition::class)
fun <T : OutputStream> fileRecordPartFormatterStep(@Named(value = "numInputPartitions") numInputPartitions: Int, partFormatter: ObjectLoaderPartFormatter<T>, @Named(value = "recordQueue") inputQueue: PartitionedQueue<PipelineEvent<StreamKey, DestinationRecordRaw>>, @Named(value = "objectLoaderPartQueue") outputQueue: PartitionedQueue<PipelineEvent<ObjectKey, ObjectLoaderPartFormatter.FormattedPart>>, taskFactory: LoadPipelineStepTaskFactory, flushStrategy: PipelineFlushStrategy): ObjectLoaderPartFormatterStep
Link copied to clipboard
@Named(value = "fileUploadCompleterStep")
@Singleton
@Requires(condition = IsFileTransferCondition::class)
fun <K : WithStream, T : RemoteObject<*>> fileUploadCompleter(objectLoader: ObjectLoader, uploadCompleter: ObjectLoaderUploadCompleter<T>, @Named(value = "fileLoadedPartQueue") inputQueue: PartitionedQueue<PipelineEvent<ObjectKey, ObjectLoaderPartLoader.PartResult<T>>>, @Named(value = "fileCompletedQueue") completedUploadQueue: PartitionedQueue<PipelineEvent<K, ObjectLoaderUploadCompleter.UploadResult<T>>>? = null, @Named(value = "fileCompletedOutputPartitioner") completedUploadPartitioner: ObjectLoaderCompletedUploadPartitioner<ObjectKey, ObjectLoaderPartLoader.PartResult<T>, K, T>? = null, taskFactory: LoadPipelineStepTaskFactory): ObjectLoaderUploadCompleterStep<K, T>
Link copied to clipboard
Link copied to clipboard
@Named(value = "oneShotObjectLoaderStep")
@Singleton
fun <O : OutputStream, K : WithStream, T : RemoteObject<*>> oneShotObjectLoaderStep(objectLoaderOneShotUploader: ObjectLoaderOneShotUploader<O, T>, @Named(value = "numInputPartitions") numInputPartitions: Int, @Named(value = "objectLoaderCompletedUploadQueue") completedUploadQueue: PartitionedQueue<PipelineEvent<K, ObjectLoaderUploadCompleter.UploadResult<T>>>? = null, completedUploadPartitioner: ObjectLoaderCompletedUploadPartitioner<StreamKey, DestinationRecordRaw, K, T>? = null, taskFactory: LoadPipelineStepTaskFactory): ObjectLoaderOneShotUploaderStep<K, T>
Link copied to clipboard
@Named(value = "recordPartFormatterStep")
@Singleton
fun <T : OutputStream> recordPartFormatter(@Named(value = "numInputPartitions") numInputPartitions: Int, partFormatter: ObjectLoaderPartFormatter<T>, @Named(value = "dataChannelInputFlows") inputFlows: Array<Flow<PipelineEvent<StreamKey, DestinationRecordRaw>>>, @Named(value = "objectLoaderPartQueue") outputQueue: PartitionedQueue<PipelineEvent<ObjectKey, ObjectLoaderPartFormatter.FormattedPart>>, taskFactory: LoadPipelineStepTaskFactory, flushStrategy: PipelineFlushStrategy): ObjectLoaderPartFormatterStep
Link copied to clipboard
@Named(value = "recordPartLoaderStep")
@Singleton
fun <T : RemoteObject<*>> recordPartLoader(loader: ObjectLoader, partLoader: ObjectLoaderPartLoader<T>, @Named(value = "objectLoaderPartQueue") inputQueue: PartitionedQueue<PipelineEvent<ObjectKey, ObjectLoaderPartFormatter.FormattedPart>>, @Named(value = "objectLoaderLoadedPartQueue") outputQueue: PartitionedQueue<PipelineEvent<ObjectKey, ObjectLoaderPartLoader.PartResult<T>>>, taskFactory: LoadPipelineStepTaskFactory): ObjectLoaderPartLoaderStep<T>
Link copied to clipboard
@Named(value = "recordUploadCompleterStep")
@Singleton
fun <K : WithStream, T : RemoteObject<*>> recordUploadCompleter(objectLoader: ObjectLoader, uploadCompleter: ObjectLoaderUploadCompleter<T>, @Named(value = "objectLoaderLoadedPartQueue") inputQueue: PartitionedQueue<PipelineEvent<ObjectKey, ObjectLoaderPartLoader.PartResult<T>>>, @Named(value = "objectLoaderCompletedUploadQueue") completedUploadQueue: PartitionedQueue<PipelineEvent<K, ObjectLoaderUploadCompleter.UploadResult<T>>>? = null, completedUploadPartitioner: ObjectLoaderCompletedUploadPartitioner<ObjectKey, ObjectLoaderPartLoader.PartResult<T>, K, T>? = null, taskFactory: LoadPipelineStepTaskFactory): ObjectLoaderUploadCompleterStep<K, T>
Link copied to clipboard
@Named(value = "uploadParallelismForSocket")
@Singleton
fun sharedUploadPermits(@Named(value = "dataChannelSocketPaths") dataChannelSocketPaths: List<String>, loader: ObjectLoader): Int