Skip to content

Instantly share code, notes, and snippets.

@martyngigg

martyngigg/docker-compose-pyiceberg.yaml

Last active October 27, 2025 11:29
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save martyngigg/290335a20efc4c851b1c3cc05990b4c6 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save martyngigg/290335a20efc4c851b1c3cc05990b4c6 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
docker-compose-pyiceberg.yaml
services:
lakekeeper-bootstrap:
image: ghcr.io/astral-sh/uv:python3.13-bookworm
entrypoint:
- "/bin/bash"
- "-c"
- |
cat > bootstrap.py <<EOF
# /// script
# requires-python = "==3.13.*"
# dependencies = [
# "requests"
# ]
# ///
import requests
resp = requests.post("http://keycloak:8080/realms/iceberg/protocol/openid-connect/token",
data={"grant_type": "client_credentials",
"client_id": "spark", "client_secret": "2OR3eRvYfSZzzZ16MlPd95jhLnOaLM52",
"scope": "lakekeeper"
}
)
resp.raise_for_status()
access_token = resp.json()["access_token"]
# create warehouse
response = requests.post("http://lakekeeper:8181/management/v1/bootstrap",
headers={
"Authorization": f"Bearer {access_token}"
},
json={"accept-terms-of-use": True})
# create warehouse
response = requests.post(
url="http://lakekeeper:8181/management/v1/warehouse",
headers={
"Authorization": f"Bearer {access_token}"
},
json={
"warehouse-name": "demo",
"storage-profile": {
"type": "s3",
"bucket": "examples",
"key-prefix": "",
"endpoint": "http://minio:9000",
"region": "local-01",
"path-style-access": True,
"flavor": "minio",
"sts-enabled": False
},
"storage-credential": {
"type": "s3",
"credential-type": "access-key",
"aws-access-key-id": "minio-root-user",
"aws-secret-access-key": "minio-root-password"
}
}
)
response = requests.get("http://lakekeeper:8181/management/v1/warehouse",
headers={
"Authorization": f"Bearer {access_token}"
},
)
print(response.json())
EOF
uv run --script bootstrap.py
networks:
- iceberg_net
volumes:
- uv_cache:/root/.cache/uv
pyiceberg-create-table:
image: ghcr.io/astral-sh/uv:python3.13-bookworm
entrypoint:
- "/bin/bash"
- "-c"
- |
cat > create-table.py <<EOF
# /// script
# requires-python = "==3.13.*"
# dependencies = [
# "pyiceberg==0.10.0",
# "pyarrow",
# "s3fs",
# ]
# ///
from pyiceberg.catalog.rest import RestCatalog
import pyarrow as pa
catalog_name = "default"
catalog_properties = {
"uri": "http://lakekeeper:8181/catalog",
"warehouse": "demo",
"credential": "spark:2OR3eRvYfSZzzZ16MlPd95jhLnOaLM52",
"oauth2-server-uri": "http://keycloak:8080/realms/iceberg/protocol/openid-connect/token",
"scope": "lakekeeper",
}
catalog = RestCatalog(catalog_name, **catalog_properties)
df = pa.Table.from_pylist(
[
{"city": "Amsterdam", "lat": 52.371807, "long": 4.896029},
{"city": "San Francisco", "lat": 37.773972, "long": -122.431297},
{"city": "Drachten", "lat": 53.11254, "long": 6.0989},
{"city": "Paris", "lat": 48.864716, "long": 2.349014},
],
)
namespace, table_name = "default", "cities"
table_id = f"{namespace}.{table_name}"
catalog.create_namespace_if_not_exists(namespace)
if catalog.table_exists(table_id):
catalog.purge_table(table_id)
tbl = catalog.create_table_if_not_exists(table_id, schema=df.schema)
tbl.append(df)
print(tbl)
EOF
uv run --script create-table.py
networks:
- iceberg_net
volumes:
- uv_cache:/root/.cache/uv
networks:
iceberg_net:
name: access-control-simple_iceberg_net
external: true
volumes:
uv_cache:
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment