bulk-cdk-core-load/io.airbyte.cdk.load.task.internal

Package-level declarations

Types

Link copied to clipboard
@Singleton
@Requires(property = "airbyte.destination.core.data-channel.medium", value = "SOCKET")
class DummyStatsMessageConsumer(consumer: OutputConsumer) : SuspendFunction1<AirbyteMessage, Unit>
Link copied to clipboard
@Factory
class FrequencyFactory
Link copied to clipboard
Link copied to clipboard
class InputConsumerTask(inputFlow: ReservingDeserializingInputFlow, pipelineInputQueue: PartitionedQueue<PipelineEvent<StreamKey, DestinationRecordRaw>>, partitioner: InputPartitioner, pipelineEventBookkeepingRouter: PipelineEventBookkeepingRouter) : Task

Routes @DestinationStreamAffinedMessages by stream to the appropriate channel and @ CheckpointMessages to the state manager.

Link copied to clipboard
class LoadPipelineStepTask<S : AutoCloseable, K1 : WithStream, T, K2 : WithStream, U : Any>(batchAccumulator: BatchAccumulator<S, K1, T, U>, inputFlow: Flow<PipelineEvent<K1, T>>, batchUpdateQueue: QueueWriter<BatchUpdate>, outputPartitioner: OutputPartitioner<K1, T, K2, U>?, outputQueue: PartitionedQueue<PipelineEvent<K2, U>>?, flushStrategy: PipelineFlushStrategy?, part: Int, numWorkers: Int, stepId: String, streamCompletions: ConcurrentHashMap<Pair<String, DestinationStream.Descriptor>, AtomicInteger>, maxNumConcurrentKeys: Int? = null) : Task

A long-running task that actually implements a load pipeline step.

Link copied to clipboard
@Singleton
@Requires(bean = LoadStrategy::class)
class LoadPipelineStepTaskFactory(@Named(value = "batchStateUpdateQueue") val batchUpdateQueue: QueueWriter<BatchUpdate>, @Named(value = "dataChannelInputFlows") val inputFlows: Array<Flow<PipelineInputEvent>>, flushStrategy: PipelineFlushStrategy)
Link copied to clipboard
@Singleton
class ReservingDeserializingInputFlow(val config: DestinationConfiguration, val deserializer: ProtocolMessageDeserializer, @Named(value = "queueMemoryManager") val memoryManager: ReservationManager, @Named(value = "inputStream") val inputStream: InputStream, @Named(value = "logPerNRecords") val logPerNRecords: Long) : Flow<Pair<Long, Reserved<DestinationMessage>>>
Link copied to clipboard
data class StateWithCounts<S : AutoCloseable>(val accumulatorState: S, val checkpointCounts: MutableMap<CheckpointId, CheckpointValue> = mutableMapOf(), val inputCount: Long = 0, val createdAtMs: Long = System.currentTimeMillis()) : AutoCloseable

Accumulator state with the checkpoint counts (Checkpoint Id -> Records Seen) seen since the state was created.

Link copied to clipboard
@Singleton
@Requires(property = "airbyte.destination.core.data-channel.medium", value = "SOCKET")
class StatsEmitter(syncManager: SyncManager, catalog: DestinationCatalog, outputConsumer: DummyStatsMessageConsumer, @Nullable @Named(value = "statsEmissionFrequencyOverride") emissionFrequencyMillis: Long? = null) : Task
Link copied to clipboard
class UpdateBatchStateTask(inputQueue: QueueReader<BatchUpdate>, syncManager: SyncManager, checkpointManager: CheckpointManager, launcher: DestinationTaskLauncher) : Task

A long-running task that updates the state of record batches after they are processed.

Link copied to clipboard
@Singleton
class UpdateBatchStateTaskFactory(@Named(value = "batchStateUpdateQueue") inputQueue: QueueReader<BatchUpdate>, syncManager: SyncManager, checkpointManager: CheckpointManager)
Link copied to clipboard
@Singleton
@Secondary
class UpdateCheckpointsTask(syncManager: SyncManager, checkpointManager: CheckpointManager, checkpointMessageQueue: MessageQueue<Reserved<CheckpointMessageWrapped>>) : Task