bulk-cdk-toolkit-extract-jdbc/io.airbyte.cdk.read

Package-level declarations

Types

data class And(val conj: List<WhereClauseNode>) : WhereClauseNode

class DefaultJdbcCursorIncrementalPartition(val selectQueryGenerator: SelectQueryGenerator, val streamState: DefaultJdbcStreamState, val cursor: Field, val cursorLowerBound: JsonNode, val isLowerBoundIncluded: Boolean, cursorUpperBound: JsonNode?) : DefaultJdbcCursorPartition

Default implementation of a JdbcPartition for a cursor incremental partition. These are always splittable.

DefaultJdbcCursorPartition

sealed class DefaultJdbcCursorPartition : DefaultJdbcSplittablePartition, JdbcCursorPartition<DefaultJdbcStreamState>

Default implementation of a JdbcPartition for a splittable partition involving cursor columns.

DefaultJdbcFetchSizeEstimator

class DefaultJdbcFetchSizeEstimator(val maxMemoryBytes: Long, val maxConcurrency: Int, val minFetchSize: Int, val defaultFetchSize: Int, val maxFetchSize: Int, val memoryCapacityRatio: Double) : JdbcSharedState.JdbcFetchSizeEstimator

FetchSizeEstimator is used to estimate the value of the JDBC fetchSize parameter to fill up a portion of the JVM heap defined by MEM_CAPACITY_RATIO.

DefaultJdbcPartition

sealed class DefaultJdbcPartition : JdbcPartition<DefaultJdbcStreamState>

Base class for default implementations of JdbcPartition.

DefaultJdbcPartitionFactory

@Singleton

class DefaultJdbcPartitionFactory(val sharedState: DefaultJdbcSharedState, val handler: CatalogValidationFailureHandler, val selectQueryGenerator: SelectQueryGenerator) : JdbcPartitionFactory<DefaultJdbcSharedState, DefaultJdbcStreamState, DefaultJdbcPartition>

Default implementation of JdbcPartitionFactory.

DefaultJdbcSharedState

@Singleton

class DefaultJdbcSharedState(val configuration: JdbcSourceConfiguration, val selectQuerier: SelectQuerier, val constants: DefaultJdbcConstants, val concurrencyResource: ConcurrencyResource, val resourceAcquirer: ResourceAcquirer) : JdbcSharedState

Default implementation of JdbcSharedState.

DefaultJdbcSplittablePartition

sealed class DefaultJdbcSplittablePartition : DefaultJdbcPartition, JdbcSplittablePartition<DefaultJdbcStreamState>

Base class for default implementations of JdbcPartition for splittable partitions.

DefaultJdbcSplittableSnapshotPartition

class DefaultJdbcSplittableSnapshotPartition(val selectQueryGenerator: SelectQueryGenerator, val streamState: DefaultJdbcStreamState, primaryKey: List<Field>, val lowerBound: List<JsonNode>?, val upperBound: List<JsonNode>?) : DefaultJdbcSplittablePartition

Default implementation of a JdbcPartition for a splittable snapshot partition.

DefaultJdbcSplittableSnapshotWithCursorPartition

class DefaultJdbcSplittableSnapshotWithCursorPartition(val selectQueryGenerator: SelectQueryGenerator, val streamState: DefaultJdbcStreamState, primaryKey: List<Field>, val lowerBound: List<JsonNode>?, val upperBound: List<JsonNode>?, val cursor: Field, cursorUpperBound: JsonNode?) : DefaultJdbcCursorPartition

Default implementation of a JdbcPartition for a splittable snapshot partition preceding a cursor-based incremental sync.

DefaultJdbcStreamState

class DefaultJdbcStreamState(val sharedState: DefaultJdbcSharedState, val streamFeedBootstrap: StreamFeedBootstrap) : JdbcStreamState<DefaultJdbcSharedState>

Default implementation of JdbcStreamState.

DefaultJdbcStreamStateValue

data class DefaultJdbcStreamStateValue(val primaryKey: Map<String, JsonNode> = mapOf(), val cursors: Map<String, JsonNode> = mapOf())

DefaultJdbcStreamStateValue is used by DefaultJdbcPartitionFactory for deserializing an OpaqueStateValue into a DefaultJdbcPartition. The latter is able to, in turn, serialize a partition boundary into an OpaqueStateValue with DefaultJdbcStreamStateValue.

DefaultJdbcUnsplittablePartition

sealed class DefaultJdbcUnsplittablePartition : DefaultJdbcPartition

Base class for default implementations of JdbcPartition for unsplittable partitions.

DefaultJdbcUnsplittableSnapshotPartition

class DefaultJdbcUnsplittableSnapshotPartition(val selectQueryGenerator: SelectQueryGenerator, val streamState: DefaultJdbcStreamState) : DefaultJdbcUnsplittablePartition

Default implementation of a JdbcPartition for an unsplittable snapshot partition.

DefaultJdbcUnsplittableSnapshotWithCursorPartition

class DefaultJdbcUnsplittableSnapshotWithCursorPartition(val selectQueryGenerator: SelectQueryGenerator, val streamState: DefaultJdbcStreamState, val cursor: Field) : DefaultJdbcUnsplittablePartition, JdbcCursorPartition<DefaultJdbcStreamState>

Default implementation of a JdbcPartition for an unsplittable snapshot partition preceding a cursor-based incremental sync.

DefaultRowByteSizeEstimator

class DefaultRowByteSizeEstimator(val estimatedRecordOverheadBytes: Long, val estimatedFieldOverheadBytes: Long) : JdbcSharedState.RowByteSizeEstimator

Estimates the in-memory byte size of a table row based on its ObjectNode representation.

Equal

data class Equal(val column: Field, val bindingValue: JsonNode) : WhereClauseLeafNode

From

data class From(val name: String, val namespace: String?) : FromNode

FromNode

sealed interface FromNode

FromSample

data class FromSample(val name: String, val namespace: String?, val sampleRateInvPow2: Int, val sampleSize: Int, val where: WhereNode? = null) : FromNode

Greater

data class Greater(val column: Field, val bindingValue: JsonNode) : WhereClauseLeafNode

GreaterOrEqual

data class GreaterOrEqual(val column: Field, val bindingValue: JsonNode) : WhereClauseLeafNode

JdbcConcurrentPartitionsCreator

class JdbcConcurrentPartitionsCreator<A : JdbcSharedState, S : JdbcStreamState<A>, P : JdbcPartition<S>>(partition: P, partitionFactory: JdbcPartitionFactory<A, S, P>) : JdbcPartitionsCreator<A, S, P>

Concurrent JDBC implementation of PartitionsCreator.

JdbcConcurrentPartitionsCreatorFactory

@Singleton

@Secondary

@Requires(property = "airbyte.connector.extract.jdbc.mode", value = "concurrent")

class JdbcConcurrentPartitionsCreatorFactory<A : JdbcSharedState, S : JdbcStreamState<A>, P : JdbcPartition<S>>(partitionFactory: JdbcPartitionFactory<A, S, P>) : JdbcPartitionsCreatorFactory<A, S, P>

Concurrent JDBC implementation of PartitionsCreatorFactory.

JdbcCursorPartition

interface JdbcCursorPartition<S : JdbcStreamState<*>> : JdbcPartition<S>

A JdbcPartition which allows cursor-based incremental reads.

JdbcNonResumablePartitionReader

class JdbcNonResumablePartitionReader>(partition: P) : JdbcPartitionReader

JDBC implementation of PartitionReader which reads the partition in its entirety.

JdbcPartition

interface JdbcPartition<S : JdbcStreamState<*>>

Encapsulates database-specific aspects relating to a JDBC stream partition, consumed by JdbcPartitionReader and friends.

JdbcPartitionCreatorFactorySupplier

@Singleton

@Requires(property = "airbyte.connector.extract.jdbc.mode", pattern = "^(sequential|concurrent)$")

class JdbcPartitionCreatorFactorySupplier<T : JdbcPartitionsCreatorFactory<A, S, P>, A : JdbcSharedState, S : JdbcStreamState<A>, P : JdbcPartition<S>>(val factory: T) : PartitionsCreatorFactorySupplier<T>

JdbcPartitionFactory

@DefaultImplementation(value = DefaultJdbcPartitionFactory::class)

interface JdbcPartitionFactory<A : JdbcSharedState, S : JdbcStreamState<A>, P : JdbcPartition<S>>

Encapsulates database-specific logic turning OpaqueStateValue into JdbcPartition.

JdbcPartitionReader

sealed class JdbcPartitionReader> : PartitionReader

Base class for JDBC implementations of PartitionReader.

JdbcPartitionsCreator

abstract class JdbcPartitionsCreator<A : JdbcSharedState, S : JdbcStreamState<A>, P : JdbcPartition<S>>(val partition: P, val partitionFactory: JdbcPartitionFactory<A, S, P>) : PartitionsCreator

Base class for JDBC implementations of PartitionsCreator.

JdbcPartitionsCreatorFactory

abstract class JdbcPartitionsCreatorFactory<A : JdbcSharedState, S : JdbcStreamState<A>, P : JdbcPartition<S>>(val partitionFactory: JdbcPartitionFactory<A, S, P>) : PartitionsCreatorFactory

Base class for JDBC implementations of PartitionsCreatorFactory.

JdbcResumablePartitionReader

class JdbcResumablePartitionReader>(partition: P) : JdbcPartitionReader

JDBC implementation of PartitionReader which reads as much as possible of the partition, in order, before timing out.

JdbcSelectQuerier

@Singleton

class JdbcSelectQuerier(jdbcConnectionFactory: JdbcConnectionFactory) : SelectQuerier

Default implementation of SelectQuerier.

JdbcSequentialPartitionsCreator

class JdbcSequentialPartitionsCreator<A : JdbcSharedState, S : JdbcStreamState<A>, P : JdbcPartition<S>>(partition: P, partitionFactory: JdbcPartitionFactory<A, S, P>) : JdbcPartitionsCreator<A, S, P>

Sequential JDBC implementation of PartitionsCreator.

JdbcSequentialPartitionsCreatorFactory

@Singleton

@Secondary

@Requires(property = "airbyte.connector.extract.jdbc.mode", value = "sequential")

class JdbcSequentialPartitionsCreatorFactory<A : JdbcSharedState, S : JdbcStreamState<A>, P : JdbcPartition<S>>(partitionFactory: JdbcPartitionFactory<A, S, P>) : JdbcPartitionsCreatorFactory<A, S, P>

Sequential JDBC implementation of PartitionsCreatorFactory.

JdbcSharedState

@DefaultImplementation(value = DefaultJdbcSharedState::class)

interface JdbcSharedState

Encapsulates database-specific state, both constant or transient, common to all partitions.

JdbcSplittablePartition

interface JdbcSplittablePartition<S : JdbcStreamState<*>> : JdbcPartition<S>

A JdbcPartition which can be subdivided.

JdbcStreamState

interface JdbcStreamState<A : JdbcSharedState>

Encapsulates database-specific transient state for a particular stream.

Lesser

data class Lesser(val column: Field, val bindingValue: JsonNode) : WhereClauseLeafNode

LesserOrEqual

data class LesserOrEqual(val column: Field, val bindingValue: JsonNode) : WhereClauseLeafNode

Limit

data class Limit(val n: Long) : LimitNode

LimitNode

sealed interface LimitNode

LimitState

data class LimitState(val predecessor: Long, val current: Long)

Adaptive LIMIT value state, where the value grows or shrinks along the Fibonacci sequence.

NoFrom

data object NoFrom : FromNode

NoLimit

data object NoLimit : LimitNode

NoOrderBy

data object NoOrderBy : OrderByNode

NoWhere

data object NoWhere : WhereNode

data class Or(val disj: List<WhereClauseNode>) : WhereClauseNode

OrderBy

data class OrderBy(val columns: List<Field>) : OrderByNode

OrderByNode

sealed interface OrderByNode

Sample

data class Sample<T>(val sampledValues: List<T>, val kind: Sample.Kind, val valueWeight: Long)

Convenience object for maintaining sampled data and its accompanying metadata.

SelectColumnMaxValue

data class SelectColumnMaxValue(val column: Field) : SelectNode

SelectColumns

data class SelectColumns(val columns: List<Field>) : SelectNode

SelectNode

sealed interface SelectNode

SelectQuerier

@DefaultImplementation(value = JdbcSelectQuerier::class)

interface SelectQuerier

SelectQuery

data class SelectQuery(val sql: String, val columns: List<Field>, val bindings: List<SelectQuery.Binding>)

Input for SelectQuerier which contains a parameterize SQL query along with its parameter bindings and a description of the columns of the result set.

SelectQueryGenerator

fun interface SelectQueryGenerator

Connector-specific stateless object which generates a valid SelectQuery for the corresponding source database.

SelectQuerySpec

data class SelectQuerySpec(val select: SelectNode, val from: FromNode = NoFrom, val where: WhereNode = NoWhere, val orderBy: OrderByNode = NoOrderBy, val limit: LimitNode = NoLimit)

Source-database-agnostic representation of a SELECT query as an abstract syntax tree. This is tailored to the subset of queries that we actually might run in a source connector.

Where

data class Where(val clause: WhereClauseNode) : WhereNode

WhereClauseLeafNode

sealed interface WhereClauseLeafNode : WhereClauseNode

WhereClauseNode

sealed interface WhereClauseNode

WhereNode

sealed interface WhereNode

Properties

MODE_PROPERTY

const val MODE_PROPERTY: String

Functions

optimize

fun OrderByNode.optimize(): OrderByNode

fun SelectNode.optimize(): SelectNode

fun SelectQuerySpec.optimize(): SelectQuerySpec

fun WhereClauseNode.optimize(): WhereClauseNode

fun WhereNode.optimize(): WhereNode