bulk-cdk-core-load/io.airbyte.cdk.load.dataflow.pipeline

Package-level declarations

Types

Link copied to clipboard
class DataFlowPipeline(input: Flow<DataFlowStageIO>, parse: DataFlowStage, aggregate: AggregateStage, flush: DataFlowStage, state: DataFlowStage, completionHandler: PipelineCompletionHandler, aggregatePublishingConfig: AggregatePublishingConfig, aggregationDispatcher: CoroutineDispatcher, flushDispatcher: CoroutineDispatcher)
Link copied to clipboard
interface DataFlowStage
Link copied to clipboard
data class DataFlowStageIO(var raw: DestinationRecordRaw? = null, var partitionKey: PartitionKey? = null, var munged: RecordDTO? = null, var aggregate: Aggregate? = null, var partitionCountsHistogram: PartitionHistogram? = null, var partitionBytesHistogram: PartitionHistogram? = null, var mappedDesc: DestinationStream.Descriptor? = null)
Link copied to clipboard
class PipelineCompletionHandler(aggStore: AggregateStore, stateHistogramStore: StateHistogramStore, statsStore: CommittedStatsStore)
Link copied to clipboard
@Singleton
class PipelineRunner(reconciler: StateReconciler, store: StateStore, pipelines: List<DataFlowPipeline>, inputStreams: ConnectorInputStreams, @Named(value = "pipelineRunnerScope") pipelineScope: CoroutineScope, @Named(value = "aggregationDispatcher") aggregationDispatcher: CoroutineDispatcher)

Orchestrates the running of pipelines in parallel, handles and propagates errors and manages the state reconciler lifecycle.