bulk-cdk-core-load/io.airbyte.cdk.load.message

Package-level declarations

Types

Represents the state of a batch of records as it moves through processing. These are generic stages for bookkeeping, which CDK interface devs can assign to processing stages as they see fit. The only significant values are

ChannelMessage

abstract class ChannelMessage<T> : Message<T>

ChannelMessageQueue

open class ChannelMessageQueue<T>(val channel: Channel<T>) : MessageQueue<T>

CheckpointMessage

sealed interface CheckpointMessage : DestinationMessage

State.

CheckpointMessageQueue

@Singleton

@Secondary

class CheckpointMessageQueue : ChannelMessageQueue<Reserved<CheckpointMessageWrapped>>

A single-channel queue for checkpoint messages. This is so updating the checkpoint manager never blocks reading from stdin.

CheckpointMessageWrapped

sealed interface CheckpointMessageWrapped

DestinationFile

data class DestinationFile(val stream: DestinationStream, val emittedAtMs: Long, val fileMessage: DestinationFile.AirbyteRecordMessageFile) : DestinationFileDomainMessage

DestinationFileDomainMessage

sealed interface DestinationFileDomainMessage : DestinationStreamAffinedMessage

DestinationFileStreamComplete

data class DestinationFileStreamComplete(val stream: DestinationStream, val emittedAtMs: Long) : DestinationFileDomainMessage

DestinationMessage

sealed interface DestinationMessage

Internal representation of destination messages. These are intended to be specialized for usability. Data should be unmarshalled to these from front-line deserialized objects.

DestinationMessageFactory

@Singleton

class DestinationMessageFactory(catalog: DestinationCatalog, @Value(value = "${airbyte.destination.core.file-transfer.enabled}") fileTransferEnabled: Boolean, @Named(value = "requireCheckpointIdOnRecordAndKeyOnState") requireCheckpointIdOnRecordAndKeyOnState: Boolean = false, namespaceMapper: NamespaceMapper, uuidGenerator: UUIDGenerator)

DestinationRecord

data class DestinationRecord(val stream: DestinationStream, val message: DestinationRecordSource, val serializedSizeBytes: Long, val checkpointId: CheckpointId? = null, val airbyteRawId: UUID) : DestinationRecordDomainMessage

DestinationRecordAirbyteValue

data class DestinationRecordAirbyteValue(val stream: DestinationStream, val data: AirbyteValue, val emittedAtMs: Long, val meta: Meta?, val serializedSizeBytes: Long = 0)

Represents a record both deserialized AND marshaled to airbyte value. The marshaling

DestinationRecordDomainMessage

sealed interface DestinationRecordDomainMessage : DestinationStreamAffinedMessage

DestinationRecordJsonSource

@JvmInline

value class DestinationRecordJsonSource(val source: AirbyteMessage) : DestinationRecordSource

DestinationRecordProtobufSource

@JvmInline

value class DestinationRecordProtobufSource(val source: AirbyteMessage.AirbyteMessageProtobuf) : DestinationRecordSource

DestinationRecordRaw

data class DestinationRecordRaw(val stream: DestinationStream, val rawData: DestinationRecordSource, val serializedSizeBytes: Long, val checkpointId: CheckpointId? = null, val airbyteRawId: UUID)

DestinationRecordSerialized

data class DestinationRecordSerialized(val stream: DestinationStream, val serialized: String)

Represents a record already in its serialized state. The intended use is for conveying records from stdin to the spill file, where reserialization is not necessary.

DestinationRecordSource

sealed interface DestinationRecordSource

A serialization-format agnostic wrapper for incoming DestinationRecord's. The record-level corollary to io.airbyte.cdk.load.data.AirbyteValueProxy, providing uniform access to both client data and metadata.

DestinationRecordStreamComplete

data class DestinationRecordStreamComplete(val stream: DestinationStream, val emittedAtMs: Long) : DestinationRecordDomainMessage

DestinationStreamAffinedMessage

sealed interface DestinationStreamAffinedMessage : DestinationMessage

Records.

EnrichedDestinationRecordAirbyteValue

data class EnrichedDestinationRecordAirbyteValue(val stream: DestinationStream, val declaredFields: LinkedHashMap<String, EnrichedAirbyteValue>, val undeclaredFields: LinkedHashMap<String, JsonNode>, val emittedAtMs: Long, val sourceMeta: Meta, val serializedSizeBytes: Long = 0, extractedAtAsTimestampWithTimezone: Boolean = false, val airbyteRawId: UUID)

FileReference

data class FileReference(val stagingFileUrl: String, val sourceFileRelativePath: String, val fileSizeBytes: Long)

FileTransferQueueEndOfStream

data class FileTransferQueueEndOfStream(val stream: DestinationStream) : FileTransferQueueMessage

FileTransferQueueMessage

sealed interface FileTransferQueueMessage

FileTransferQueueRecord

data class FileTransferQueueRecord(val stream: DestinationStream, val file: DestinationFile, val index: Long, val checkpointId: CheckpointId) : FileTransferQueueMessage

GlobalCheckpoint

data class GlobalCheckpoint(val state: JsonNode?, val sourceStats: CheckpointMessage.Stats?, var destinationStats: CheckpointMessage.Stats? = null, val checkpoints: List<CheckpointMessage.Checkpoint> = emptyList(), val additionalProperties: Map<String, Any>, val originalTypeField: AirbyteStateMessage.AirbyteStateType? = AirbyteStateMessage.AirbyteStateType.GLOBAL, val serializedSizeBytes: Long, val checkpointKey: CheckpointKey? = null, var totalRecords: Long? = null, var totalBytes: Long? = null, var totalRejectedRecords: Long? = null, var additionalStats: MutableMap<String, Double> = mutableMapOf()) : CheckpointMessage

GlobalCheckpointWrapped

data class GlobalCheckpointWrapped(val checkpointKey: CheckpointKey, val checkpoint: CheckpointMessage) : CheckpointMessageWrapped

GlobalSnapshotCheckpoint

data class GlobalSnapshotCheckpoint(val state: JsonNode?, val sourceStats: CheckpointMessage.Stats?, var destinationStats: CheckpointMessage.Stats? = null, val checkpoints: List<CheckpointMessage.Checkpoint> = emptyList(), val additionalProperties: Map<String, Any>, val originalTypeField: AirbyteStateMessage.AirbyteStateType? = AirbyteStateMessage.AirbyteStateType.GLOBAL, val serializedSizeBytes: Long, val checkpointKey: CheckpointKey? = null, var totalRecords: Long? = null, var totalBytes: Long? = null, var totalRejectedRecords: Long? = null, var additionalStats: MutableMap<String, Double> = mutableMapOf(), val streamCheckpoints: Map<DestinationStream.Descriptor, CheckpointKey>) : CheckpointMessage

GlobalSnapshotCheckpointWrapped

data class GlobalSnapshotCheckpointWrapped(val checkpointKey: CheckpointKey, val checkpoint: CheckpointMessage) : CheckpointMessageWrapped

Ignored

data object Ignored : DestinationMessage

For messages, we recognize but do not want to process. Different from Undefined mainly in that we don't log a warning.

InputCheckpoint

sealed interface InputCheckpoint : InputMessage

InputFile

data class InputFile(val file: DestinationFile) : InputMessage

InputGlobalCheckpoint

data class InputGlobalCheckpoint(val sharedState: JsonNode?, val checkpointKey: CheckpointKey? = null, val streamStates: List<CheckpointMessage.Checkpoint> = emptyList(), val sourceRecordCount: Long? = null) : InputCheckpoint

InputMessage

sealed interface InputMessage

InputMessageOther

data class InputMessageOther(val airbyteMessage: AirbyteMessage) : InputMessage

InputRecord

data class InputRecord(val stream: DestinationStream, val data: AirbyteValue, val emittedAtMs: Long, val meta: Meta?, val serialized: String, val fileReference: AirbyteRecordMessageFileReference? = null, val checkpointId: CheckpointId? = null, val unknownFieldNames: Set<String> = emptySet()) : InputMessage

InputStreamCheckpoint

data class InputStreamCheckpoint(val checkpoint: StreamCheckpoint) : InputCheckpoint

InputStreamComplete

data class InputStreamComplete(val streamComplete: DestinationRecordStreamComplete) : InputMessage

Message

interface Message<T> : MessageReader<T> , MessageWriter<T>

MessageQueue

interface MessageQueue<T> : QueueReader<T> , QueueWriter<T>

MessageReader

interface MessageReader<T>

MessageSupplier

interface MessageSupplier<K, T>

MessageWriter

interface MessageWriter<T> : CloseableCoroutine