AWS Glue catalog

Connect RisingWave to an AWS Glue Data Catalog for Apache Iceberg.

You can connect RisingWave to an AWS Glue Data Catalog to manage metadata for your Iceberg tables. This allows you to use Glue as a central metastore when creating an Iceberg SOURCE, SINK, or CONNECTION (for internal Iceberg tables).

For a complete list of all catalog-related parameters, see the main Catalog configuration page.

Example

Examples for using an AWS Glue catalog with internal Iceberg tables, sinks, and sources.

Internal tables (recommended)

Use Glue as the catalog for RisingWave-managed (internal) Iceberg tables by creating an Iceberg CONNECTION, then creating tables with ENGINE = iceberg.

CREATE CONNECTION glue_conn WITH (
    type = 'iceberg',

    warehouse.path = 's3://my-bucket/warehouse/',

    catalog.type = 'glue',
    catalog.name = 'my_glue_catalog',
    glue.region = 'us-west-2',

    s3.region = 'us-west-2',
    s3.access.key = '...',
    s3.secret.key = '...',
    enable_config_load = false
);

SET iceberg_engine_connection = 'public.glue_conn';

-- `commit_checkpoint_interval` controls Iceberg commit frequency. Default: about every 60 seconds; set to 1 for faster commits and visibility.
CREATE TABLE my_internal_iceberg_table (
  id INT PRIMARY KEY,
  name VARCHAR
)
WITH (commit_checkpoint_interval = 1)
ENGINE = iceberg;

Write (CREATE SINK)

Deliver data from RisingWave to a Glue-managed Iceberg table using CREATE SINK.

CREATE SINK my_glue_sink FROM my_mv WITH (
    connector = 'iceberg',
    type = 'upsert',
    primary_key = 'id',

    warehouse.path = 's3://my-bucket/warehouse/',
    database.name = 'my_db',
    create_table_if_not_exists = 'true',
    table.name = 'my_table',

    catalog.type = 'glue',
    catalog.name = 'my_glue_catalog',
    glue.region = 'us-west-2',

    s3.region = 'us-west-2',
    s3.access.key = '...',
    s3.secret.key = '...',
    enable_config_load = false
);

Read (CREATE SOURCE)

Ingest data from a Glue-managed Iceberg table into RisingWave using CREATE SOURCE.

CREATE SOURCE my_glue_source WITH (
    connector = 'iceberg',

    warehouse.path = 's3://my-bucket/warehouse/',
    database.name = 'my_db',
    table.name = 'my_table',

    catalog.type = 'glue',
    catalog.name = 'my_glue_catalog',
    glue.region = 'us-west-2',

    s3.region = 'us-west-2',
    s3.access.key = '...',
    s3.secret.key = '...',
    enable_config_load = false
);

Optional: Use separate permissions for S3 and Glue

Support for glue.* IAM role parameters was added in v2.7.0.

By default, if Glue-specific credentials/role are not provided, RisingWave falls back to using the S3 credentials/role.

If you want to use different permissions for S3 and Glue, you can configure them separately. This still follows the same two authentication methods as above:

Internal tables (CONNECTION)

-- Option 1 (separate AK/SK): enable_config_load = false
CREATE CONNECTION glue_conn WITH (
    type = 'iceberg',
    catalog.type = 'glue',
    catalog.name = 'demo',
    warehouse.path = 's3://my-bucket/warehouse/',

    -- S3 credentials
    s3.region = 'ap-southeast-2',
    s3.access.key = '...',
    s3.secret.key = '...',

    -- Glue credentials (optional; otherwise falls back to S3)
    glue.region = 'ap-southeast-2',
    glue.access.key = '...',
    glue.secret.key = '...',

    enable_config_load = false
);

-- Option 2 (separate IAM roles): enable_config_load = true
-- Prereq: grant Glue and S3 permissions to your IAM roles, then make sure the machine running this SQL can assume the roles (via env/config),
-- or specify the roles explicitly with s3.iam_role_arn and glue.iam_role_arn.
CREATE CONNECTION glue_conn WITH (
    type = 'iceberg',
    catalog.type = 'glue',
    catalog.name = 'demo',
    warehouse.path = 's3://my-bucket/warehouse/',

    -- S3 role (optional if already in env/config)
    s3.region = 'ap-southeast-2',
    s3.iam_role_arn = 'arn:aws:iam::123456789012:role/MyS3Role',

    -- Glue role (optional; otherwise falls back to S3 role)
    glue.region = 'ap-southeast-2',
    glue.iam_role_arn = 'arn:aws:iam::123456789012:role/MyGlueRole',

    enable_config_load = true
);

Write (CREATE SINK)

-- Option 1 (separate AK/SK): enable_config_load = false
CREATE SINK sink_t FROM t WITH (
    connector = 'iceberg',
    type = 'append-only',
    force_append_only = 'true',

    catalog.type = 'glue',
    catalog.name = 'demo',
    warehouse.path = 's3://my-bucket/warehouse/',
    database.name = 'demo',
    create_table_if_not_exists = 'true',
    table.name = 't',

    -- S3 credentials
    s3.region = 'ap-southeast-2',
    s3.access.key = '...',
    s3.secret.key = '...',

    -- Glue credentials (optional; otherwise falls back to S3)
    glue.region = 'ap-southeast-2',
    glue.access.key = '...',
    glue.secret.key = '...',

    enable_config_load = false
);

-- Option 2 (separate IAM roles): enable_config_load = true
-- Prereq: grant Glue and S3 permissions to your IAM roles, then make sure the machine running this SQL can assume the roles (via env/config),
-- or specify the roles explicitly with s3.iam_role_arn and glue.iam_role_arn.
CREATE SINK sink_t FROM t WITH (
    connector = 'iceberg',
    type = 'append-only',
    force_append_only = 'true',

    catalog.type = 'glue',
    catalog.name = 'demo',
    warehouse.path = 's3://my-bucket/warehouse/',
    database.name = 'demo',
    create_table_if_not_exists = 'true',
    table.name = 't',

    -- S3 role (optional if already in env/config)
    s3.region = 'ap-southeast-2',
    s3.iam_role_arn = 'arn:aws:iam::123456789012:role/MyS3Role',

    -- Glue role (optional; otherwise falls back to S3 role)
    glue.region = 'ap-southeast-2',
    glue.iam_role_arn = 'arn:aws:iam::123456789012:role/MyGlueRole',

    enable_config_load = true
);

Read (CREATE SOURCE)

-- Option 1 (separate AK/SK): enable_config_load = false
CREATE SOURCE source_t WITH (
    connector = 'iceberg',
    catalog.type = 'glue',
    catalog.name = 'demo',
    warehouse.path = 's3://my-bucket/warehouse/',
    database.name = 'demo',
    table.name = 't',

    -- S3 credentials
    s3.region = 'ap-southeast-2',
    s3.access.key = '...',
    s3.secret.key = '...',

    -- Glue credentials (optional; otherwise falls back to S3)
    glue.region = 'ap-southeast-2',
    glue.access.key = '...',
    glue.secret.key = '...',

    enable_config_load = false
);

-- Option 2 (separate IAM roles): enable_config_load = true
-- Prereq: grant Glue and S3 permissions to your IAM roles, then make sure the machine running this SQL can assume the roles (via env/config),
-- or specify the roles explicitly with s3.iam_role_arn and glue.iam_role_arn.
CREATE SOURCE source_t WITH (
    connector = 'iceberg',
    catalog.type = 'glue',
    catalog.name = 'demo',
    warehouse.path = 's3://my-bucket/warehouse/',
    database.name = 'demo',
    table.name = 't',

    -- S3 role (optional if already in env/config)
    s3.region = 'ap-southeast-2',
    s3.iam_role_arn = 'arn:aws:iam::123456789012:role/MyS3Role',

    -- Glue role (optional; otherwise falls back to S3 role)
    glue.region = 'ap-southeast-2',
    glue.iam_role_arn = 'arn:aws:iam::123456789012:role/MyGlueRole',

    enable_config_load = true
);