bulk-cdk-toolkit-load-iceberg-parquet/io.airbyte.cdk.load.toolkits.iceberg.parquet.io/IcebergUtil

IcebergUtil

@Singleton

class IcebergUtil(tableIdGenerator: TableIdGenerator, coercer: AirbyteValueCoercer)

Constructors

IcebergUtil

constructor(tableIdGenerator: TableIdGenerator, coercer: AirbyteValueCoercer)

Functions

constructGenerationIdSuffix

fun constructGenerationIdSuffix(stream: DestinationStream): String

fun constructGenerationIdSuffix(generationId: Long): String

createCatalog

fun createCatalog(catalogName: String, properties: Map<String, String>): Catalog

Builds an Iceberg Catalog.

createNamespace

fun createNamespace(streamDescriptor: DestinationStream.Descriptor, catalog: Catalog)

Create the namespace if it doesn't already exist.

createTable

fun createTable(streamDescriptor: DestinationStream.Descriptor, catalog: Catalog, schema: Schema): Table

Builds (if necessary) an Iceberg Table. This includes creating the table's namespace if it does not already exist. If the Table already exists, it is loaded from the Catalog.

getOperation

fun getOperation(fields: Map<String, AirbyteValue>, importType: ImportType): Operation

mungeForIceberg

fun mungeForIceberg(value: EnrichedAirbyteValue): EnrichedAirbyteValue

toIcebergRecord

fun toIcebergRecord(fields: Map<String, AirbyteValue>, icebergSchema: Schema): GenericRecord

toIcebergSchema

fun toIcebergSchema(stream: DestinationStream): Schema

transformValueRecursingIntoArrays

fun EnrichedAirbyteValue.transformValueRecursingIntoArrays(transformer: (AirbyteValue, AirbyteType) -> ChangedValue?): EnrichedAirbyteValue

RB: I just moved this inside the class. I do not take ownership of this code.