Airbyte Kotlin Bulk CDK

0.1.92

bulk-cdk-toolkit-load-iceberg-parquet/io.airbyte.cdk.load.toolkits.iceberg.parquet.io/IcebergUtil

IcebergUtil

@Singleton

class IcebergUtil(tableIdGenerator: TableIdGenerator)

Constructors

constructor(tableIdGenerator: TableIdGenerator)

Types

InvalidFormatException

class InvalidFormatException(message: String) : Exception

Functions

assertGenerationIdSuffixIsOfValidFormat

fun assertGenerationIdSuffixIsOfValidFormat(generationId: String)

constructGenerationIdSuffix

fun constructGenerationIdSuffix(stream: DestinationStream): String

fun constructGenerationIdSuffix(generationId: Long): String

fun createCatalog(catalogName: String, properties: Map<String, String>): Catalog

Builds an Iceberg Catalog.

createNamespace

fun createNamespace(streamDescriptor: DestinationStream.Descriptor, catalog: Catalog)

Create the namespace if it doesn't already exist.

fun createTable(streamDescriptor: DestinationStream.Descriptor, catalog: Catalog, schema: Schema): Table

Builds (if necessary) an Iceberg Table. This includes creating the table's namespace if it does not already exist. If the Table already exists, it is loaded from the Catalog.

fun getOperation(record: EnrichedDestinationRecordAirbyteValue, importType: ImportType): Operation

toIcebergSchema

fun toIcebergSchema(stream: DestinationStream): Schema

fun toRecord(record: EnrichedDestinationRecordAirbyteValue, stream: DestinationStream, tableSchema: Schema): Record

Converts an Airbyte EnrichedDestinationRecordAirbyteValue into an Iceberg Record. The converted record will be wrapped to include Operation information, which is used by the writer to determine how to write the data to the underlying Iceberg files.