Airbyte Kotlin Bulk CDK

0.1.92

bulk-cdk-core-load/io.airbyte.cdk.load.state

Package-level declarations

Types

value class CheckpointId(val value: String)

Uniquely identifies the checkpoint. This is used by the StreamManager to count persisted records against the checkpoint. Specifically, it should be passed to the StreamManager to determine data sufficiency.

CheckpointIndex

value class CheckpointIndex(val value: Int)

Represents the checkpoint's order (stream-level for stream state, global for global state). Specifically, no state shall be released for CheckpointIndex N until all state for CheckpointIndexes 1..N-1 have been released.

data class CheckpointKey(val checkpointIndex: CheckpointIndex, val checkpointId: CheckpointId) : Comparable<CheckpointKey>

Used internally by the checkpoint manager to maintain ordered maps of checkpoints. Ordered by index only.

CheckpointManager

class CheckpointManager(val catalog: DestinationCatalog, val syncManager: SyncManager, val outputConsumer: suspend (Reserved<CheckpointMessage>, Long, Long, Long) -> Unit, val timeProvider: TimeProvider, val socketMode: Boolean, val namespaceMapper: NamespaceMapper)

Message-type agnostic streams checkpoint manager.

CheckpointValue

data class CheckpointValue(val records: Long, val serializedBytes: Long, val rejectedRecords: Long = 0)

For tracking counts against checkpoints of records read and persisted. Currently it only tracks row count, but could be extended to track bytes moved if needed.

DefaultDestinationStateManager

@Singleton

@Secondary

class DefaultDestinationStateManager<T : DestinationState>(persister: DestinationStatePersister<T>) : DestinationStateManager<T>

DestinationFailure

data class DestinationFailure(val cause: Exception, val streamResults: Map<DestinationStream.Descriptor, StreamResult>) : DestinationResult

DestinationResult

sealed interface DestinationResult

DestinationState

interface DestinationState

DestinationStateManager

interface DestinationStateManager<T : DestinationState>

DestinationStatePersister

interface DestinationStatePersister<T : DestinationState>

DestinationSuccess

data object DestinationSuccess : DestinationResult

FreeingAnnotatingCheckpointConsumer

@Singleton

class FreeingAnnotatingCheckpointConsumer(consumer: OutputConsumer) : SuspendFunction4<Reserved<CheckpointMessage>, Long, Long, Long, Unit>

PipelineEventBookkeepingRouter

@Singleton

class PipelineEventBookkeepingRouter(catalog: DestinationCatalog, syncManager: SyncManager, checkpointQueue: QueueWriter<Reserved<CheckpointMessageWrapped>>, openStreamQueue: QueueWriter<DestinationStream>, fileTransferQueue: MessageQueue<FileTransferQueueMessage>, @Named(value = "batchStateUpdateQueue") batchStateUpdateQueue: ChannelMessageQueue<BatchUpdate>, @Named(value = "numDataChannels") numDataChannels: Int, @Named(value = "markEndOfStreamAtEndOfSync") markEndOfStreamAtEndOfSync: Boolean, namespaceMapper: NamespaceMapper) : CloseableCoroutine

The stdio and socket input channel differ in subtle and critical ways

ReservationManager

class ReservationManager(val totalCapacityBytes: Long)

Manages memory usage for the destination.

data class Reserved<T>(parentManager: ReservationManager? = null, val bytesReserved: Long = 0, val value: T) : CloseableCoroutine

Releasable reservation of memory.

class StreamManager(val stream: DestinationStream)

Manages the state of a single stream.

StreamProcessingFailed

data class StreamProcessingFailed(val streamException: Exception) : StreamResult

StreamProcessingSucceeded

data object StreamProcessingSucceeded : StreamResult

sealed interface StreamResult

@Singleton

class SyncManager(val catalog: DestinationCatalog)