bulk-cdk-toolkit-load-object-storage/io.airbyte.cdk.load.pipline.object_storage.file/FileChunkStep

FileChunkStep

@Singleton
@Requires(condition = IsFileTransferCondition::class)
class FileChunkStep<T : RemoteObject<*>>(catalog: DestinationCatalog, fileLoader: ObjectLoader, @Named(value = "fileQueue") inputQueue: PartitionedQueue<PipelineEvent<StreamKey, DestinationRecordRaw>>, @Named(value = "filePartQueue") partQueue: PartitionedQueue<PipelineEvent<ObjectKey, ObjectLoaderPartFormatter.FormattedPart>>, pathFactory: ObjectStoragePathFactory, uploadIdGenerator: UploadIdGenerator) : LoadPipelineStep

Constructors

Link copied to clipboard
constructor(catalog: DestinationCatalog, fileLoader: ObjectLoader, @Named(value = "fileQueue") inputQueue: PartitionedQueue<PipelineEvent<StreamKey, DestinationRecordRaw>>, @Named(value = "filePartQueue") partQueue: PartitionedQueue<PipelineEvent<ObjectKey, ObjectLoaderPartFormatter.FormattedPart>>, pathFactory: ObjectStoragePathFactory, uploadIdGenerator: UploadIdGenerator)

Properties

Link copied to clipboard
open override val numWorkers: Int = 1

Functions

Link copied to clipboard
open override fun taskForPartition(partition: Int): Task