Skip to content

Instantly share code, notes, and snippets.

@alxthm

alxthm/icechunk-virtualizarr-minimal-reproducible-example.ipynb

Last active August 20, 2025 11:32
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save alxthm/90f6a76d458eb64be4e56f95106f18ad to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save alxthm/90f6a76d458eb64be4e56f95106f18ad to your computer and use it in GitHub Desktop.
Download ZIP
Minimal reproducible example notebook (icechunk + virtualizarr + distributed dask / deserialization error)
Display the source blob
Display the rendered blob
Raw
icechunk-virtualizarr-minimal-reproducible-example.ipynb
{
"cells": [
{
"cell_type": "markdown",
"id": "97d0010a-c60a-460a-b1ff-b2d7555988b0",
"metadata": {},
"source": [
"# Minimal reproducible example: icechunk + virtualizarr + distributed dask deserialization error\n",
"\n",
"Environment: python 3.12 (macOS, arm)\n",
"\n",
"```\n",
"pip install 'virtualizarr['icechunk','hdf']' ipykernel dask distributed pooch scipy netcdf4 cftime\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "c5c71c89-e71d-482e-aef4-c47c168f64c4",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" \u001b[2m2025-08-20T11:31:30.191092Z\u001b[0m \u001b[33m WARN\u001b[0m \u001b[1;33micechunk::storage::object_store\u001b[0m\u001b[33m: \u001b[33mThe LocalFileSystem storage is not safe for concurrent commits. If more than one thread/process will attempt to commit at the same time, prefer using object stores.\u001b[0m\n",
" \u001b[2;3mat\u001b[0m icechunk/src/storage/object_store.rs:80\n",
"\n"
]
}
],
"source": [
"import distributed\n",
"import icechunk.xarray\n",
"import xarray as xr\n",
"import virtualizarr as vz\n",
"import obstore\n",
"\n",
"LOCAL_ICECHUNK_STORAGE = '../data/icechunk-local-icechunk-dask-test'\n",
"LOCAL_PREFIX = \"file:///Users/\"\n",
"\n",
"storage = icechunk.local_filesystem_storage(LOCAL_ICECHUNK_STORAGE)\n",
"config = icechunk.RepositoryConfig.default()\n",
"config.set_virtual_chunk_container(\n",
" icechunk.VirtualChunkContainer(\n",
" url_prefix=LOCAL_PREFIX,\n",
" store=icechunk.local_filesystem_store(LOCAL_ICECHUNK_STORAGE),\n",
" ),\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "0f4c11de-29da-4a39-9fe5-643aba4fe40a",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1.1.3\n"
]
}
],
"source": [
"print(icechunk.__version__)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "98e1c30b-319e-417d-a053-206c57029d24",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2.1.1\n"
]
}
],
"source": [
"print(vz.__version__)"
]
},
{
"cell_type": "markdown",
"id": "13c5ea5a-dbb9-4133-819a-e3dfbc4b427b",
"metadata": {},
"source": [
"## 1. Setup store"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "b4efca66-54c8-4a9b-8d93-2f5f157ed302",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'9XTR5ZX5P2ZHZWBGNZV0'"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"repo = icechunk.Repository.open_or_create(storage, config, {})\n",
"session = repo.writable_session(\"main\")\n",
"\n",
"(xr\n",
" .tutorial.open_dataset(\"rasm\", chunks={\"time\": 1})\n",
" .isel(time=slice(24))\n",
" .to_netcdf('./rasm-test.nc'))\n",
"\n",
"parser = vz.parsers.HDFParser()\n",
"store = obstore.store.LocalStore()\n",
"registry = vz.registry.ObjectStoreRegistry({ LOCAL_PREFIX : store})\n",
"\n",
"vds = vz.open_virtual_dataset('./rasm-test.nc', registry, parser)\n",
"\n",
"vds.vz.to_icechunk(session.store)\n",
"session.commit(\"Save example virtualizarr Dataset\")"
]
},
{
"cell_type": "markdown",
"id": "8a9a2222-793b-40e1-bcde-495eddb213a5",
"metadata": {},
"source": [
"## 2. Access and use dask"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "5068fe4c-7e54-4cf6-a0fb-ddf9753d2045",
"metadata": {},
"outputs": [],
"source": [
"repo = icechunk.Repository.open(storage, config, authorize_virtual_chunk_access={LOCAL_PREFIX: None})\n",
"session = repo.readonly_session(\"main\")\n",
"\n",
"ds = xr.open_dataset(session.store, engine=\"zarr\", chunks={}, zarr_format=3)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "619e64c9-80a8-4131-94d5-8dece81a59fb",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
" *\n",
" */\n",
"\n",
":root {\n",
" --xr-font-color0: var(\n",
" --jp-content-font-color0,\n",
" var(--pst-color-text-base rgba(0, 0, 0, 1))\n",
" );\n",
" --xr-font-color2: var(\n",
" --jp-content-font-color2,\n",
" var(--pst-color-text-base, rgba(0, 0, 0, 0.54))\n",
" );\n",
" --xr-font-color3: var(\n",
" --jp-content-font-color3,\n",
" var(--pst-color-text-base, rgba(0, 0, 0, 0.38))\n",
" );\n",
" --xr-border-color: var(\n",
" --jp-border-color2,\n",
" hsl(from var(--pst-color-on-background, white) h s calc(l - 10))\n",
" );\n",
" --xr-disabled-color: var(\n",
" --jp-layout-color3,\n",
" hsl(from var(--pst-color-on-background, white) h s calc(l - 40))\n",
" );\n",
" --xr-background-color: var(\n",
" --jp-layout-color0,\n",
" var(--pst-color-on-background, white)\n",
" );\n",
" --xr-background-color-row-even: var(\n",
" --jp-layout-color1,\n",
" hsl(from var(--pst-color-on-background, white) h s calc(l - 5))\n",
" );\n",
" --xr-background-color-row-odd: var(\n",
" --jp-layout-color2,\n",
" hsl(from var(--pst-color-on-background, white) h s calc(l - 15))\n",
" );\n",
"}\n",
"\n",
"html[theme=\"dark\"],\n",
"html[data-theme=\"dark\"],\n",
"body[data-theme=\"dark\"],\n",
"body.vscode-dark {\n",
" --xr-font-color0: var(\n",
" --jp-content-font-color0,\n",
" var(--pst-color-text-base, rgba(255, 255, 255, 1))\n",
" );\n",
" --xr-font-color2: var(\n",
" --jp-content-font-color2,\n",
" var(--pst-color-text-base, rgba(255, 255, 255, 0.54))\n",
" );\n",
" --xr-font-color3: var(\n",
" --jp-content-font-color3,\n",
" var(--pst-color-text-base, rgba(255, 255, 255, 0.38))\n",
" );\n",
" --xr-border-color: var(\n",
" --jp-border-color2,\n",
" hsl(from var(--pst-color-on-background, #111111) h s calc(l + 10))\n",
" );\n",
" --xr-disabled-color: var(\n",
" --jp-layout-color3,\n",
" hsl(from var(--pst-color-on-background, #111111) h s calc(l + 40))\n",
" );\n",
" --xr-background-color: var(\n",
" --jp-layout-color0,\n",
" var(--pst-color-on-background, #111111)\n",
" );\n",
" --xr-background-color-row-even: var(\n",
" --jp-layout-color1,\n",
" hsl(from var(--pst-color-on-background, #111111) h s calc(l + 5))\n",
" );\n",
" --xr-background-color-row-odd: var(\n",
" --jp-layout-color2,\n",
" hsl(from var(--pst-color-on-background, #111111) h s calc(l + 15))\n",
" );\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block !important;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-array-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 0 20px 0 20px;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: inline-block;\n",
" opacity: 0;\n",
" height: 0;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
" border: 2px solid transparent !important;\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:focus + label {\n",
" border: 2px solid var(--xr-font-color0) !important;\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: \"►\";\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: \"▼\";\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
" padding-bottom: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: \"(\";\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: \")\";\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: \",\";\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" border-color: var(--xr-background-color-row-odd);\n",
" margin-bottom: 0;\n",
" padding-top: 2px;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
" border-color: var(--xr-background-color-row-even);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-index-preview {\n",
" grid-column: 2 / 5;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" display: none;\n",
" border-top: 2px dotted var(--xr-background-color);\n",
" padding-bottom: 20px !important;\n",
" padding-top: 10px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in + label,\n",
".xr-var-data-in + label,\n",
".xr-index-data-in + label {\n",
" padding: 0 1px;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data,\n",
".xr-index-data-in:checked ~ .xr-index-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-data > pre,\n",
".xr-index-data > pre,\n",
".xr-var-data > table > tbody > tr {\n",
" background-color: transparent !important;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-index-name div,\n",
".xr-index-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2,\n",
".xr-no-icon {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked + label > .xr-icon-file-text2,\n",
".xr-var-data-in:checked + label > .xr-icon-database,\n",
".xr-index-data-in:checked + label > .xr-icon-database {\n",
" color: var(--xr-font-color0);\n",
" filter: drop-shadow(1px 1px 5px var(--xr-font-color2));\n",
" stroke-width: 0.8px;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.DataArray &#x27;Tair&#x27; ()&gt; Size: 8B\n",
"array(329287)</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.DataArray</div><div class='xr-array-name'>'Tair'</div></div><ul class='xr-sections'><li class='xr-section-item'><div class='xr-array-wrap'><input id='section-0ee3c1cf-c8c1-4fc8-8552-b9d683cfa647' class='xr-array-in' type='checkbox' checked><label for='section-0ee3c1cf-c8c1-4fc8-8552-b9d683cfa647' title='Show/hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-array-preview xr-preview'><span>329287</span></div><div class='xr-array-data'><pre>array(329287)</pre></div></div></li><li class='xr-section-item'><input id='section-6b1e2f0a-29df-479c-9629-ee7804efb9a3' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-6b1e2f0a-29df-479c-9629-ee7804efb9a3' class='xr-section-summary' title='Expand/collapse section'>Coordinates: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'></ul></div></li><li class='xr-section-item'><input id='section-2d47bb9f-41a2-4b90-9187-23ba5a6cc9ed' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-2d47bb9f-41a2-4b90-9187-23ba5a6cc9ed' class='xr-section-summary' title='Expand/collapse section'>Indexes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'></ul></div></li><li class='xr-section-item'><input id='section-f21b0d19-f61d-4790-99d6-e1a41acddfd0' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-f21b0d19-f61d-4790-99d6-e1a41acddfd0' class='xr-section-summary' title='Expand/collapse section'>Attributes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.DataArray 'Tair' ()> Size: 8B\n",
"array(329287)"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"x = (ds.Tair > 0).sum()\n",
"x.load()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "63946ee6-d8f3-4f36-860f-7043553d38d1",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2025-08-20 13:31:34,834 - distributed.scheduler - ERROR - Error during deserialization of the task graph. This frequently\n",
"occurs if the Scheduler and Client have different environments.\n",
"For more information, see\n",
"https://docs.dask.org/en/stable/deployment-considerations.html#consistent-software-environments\n",
"\n"
]
}
],
"source": [
"cluster = distributed.LocalCluster()\n",
"client = distributed.Client(cluster)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "fef4b05a-aaa6-4550-9174-8f2c8e96c29b",
"metadata": {},
"outputs": [
{
"ename": "RuntimeError",
"evalue": "Error during deserialization of the task graph. This frequently\noccurs if the Scheduler and Client have different environments.\nFor more information, see\nhttps://docs.dask.org/en/stable/deployment-considerations.html#consistent-software-environments\n",
"output_type": "error",
"traceback": [
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
"\u001b[31mValueError\u001b[39m Traceback (most recent call last)",
"\u001b[36mFile \u001b[39m\u001b[32m~/dev/work/azard/hacking-around/poc-climate-data/notebooks/icechunk-virtualizarr-minimal-reproducible-example-venv/lib/python3.12/site-packages/distributed/scheduler.py:4874\u001b[39m, in \u001b[36mupdate_graph\u001b[39m\u001b[34m()\u001b[39m\n\u001b[32m 4873\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m4874\u001b[39m expr = deserialize(expr_ser.header, expr_ser.frames)\n\u001b[32m 4875\u001b[39m \u001b[38;5;28;01mdel\u001b[39;00m expr_ser\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/dev/work/azard/hacking-around/poc-climate-data/notebooks/icechunk-virtualizarr-minimal-reproducible-example-venv/lib/python3.12/site-packages/distributed/protocol/serialize.py:452\u001b[39m, in \u001b[36mdeserialize\u001b[39m\u001b[34m()\u001b[39m\n\u001b[32m 451\u001b[39m dumps, loads, wants_context = families[name]\n\u001b[32m--> \u001b[39m\u001b[32m452\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m loads(header, frames)\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/dev/work/azard/hacking-around/poc-climate-data/notebooks/icechunk-virtualizarr-minimal-reproducible-example-venv/lib/python3.12/site-packages/distributed/protocol/serialize.py:111\u001b[39m, in \u001b[36mpickle_loads\u001b[39m\u001b[34m()\u001b[39m\n\u001b[32m 106\u001b[39m buffers = [\n\u001b[32m 107\u001b[39m ensure_writeable_flag(ensure_memoryview(mv), w)\n\u001b[32m 108\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m mv, w \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(buffers, header[\u001b[33m\"\u001b[39m\u001b[33mwriteable\u001b[39m\u001b[33m\"\u001b[39m])\n\u001b[32m 109\u001b[39m ]\n\u001b[32m--> \u001b[39m\u001b[32m111\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m pickle.loads(pik, buffers=buffers)\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/dev/work/azard/hacking-around/poc-climate-data/notebooks/icechunk-virtualizarr-minimal-reproducible-example-venv/lib/python3.12/site-packages/distributed/protocol/pickle.py:95\u001b[39m, in \u001b[36mloads\u001b[39m\u001b[34m()\u001b[39m\n\u001b[32m 94\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m95\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m pickle.loads(x)\n\u001b[32m 96\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/dev/work/azard/hacking-around/poc-climate-data/notebooks/icechunk-virtualizarr-minimal-reproducible-example-venv/lib/python3.12/site-packages/icechunk/store.py:88\u001b[39m, in \u001b[36m__setstate__\u001b[39m\u001b[34m()\u001b[39m\n\u001b[32m 87\u001b[39m store_repr = state[\u001b[33m\"\u001b[39m\u001b[33m_store\u001b[39m\u001b[33m\"\u001b[39m]\n\u001b[32m---> \u001b[39m\u001b[32m88\u001b[39m state[\u001b[33m\"\u001b[39m\u001b[33m_store\u001b[39m\u001b[33m\"\u001b[39m] = PyStore.from_bytes(store_repr)\n\u001b[32m 89\u001b[39m \u001b[38;5;28mself\u001b[39m.\u001b[34m__dict__\u001b[39m = state\n",
"\u001b[31mValueError\u001b[39m: Failed to deserialize store from bytes: deserialization error\n\ncontext:\n 0: icechunk::store::from_bytes\n at icechunk/src/store.rs:161\n",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[31mRuntimeError\u001b[39m Traceback (most recent call last)",
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[8]\u001b[39m\u001b[32m, line 2\u001b[39m\n\u001b[32m 1\u001b[39m x = (ds.Tair > \u001b[32m0\u001b[39m).sum()\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m \u001b[43mx\u001b[49m\u001b[43m.\u001b[49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/dev/work/azard/hacking-around/poc-climate-data/notebooks/icechunk-virtualizarr-minimal-reproducible-example-venv/lib/python3.12/site-packages/xarray/core/dataarray.py:1167\u001b[39m, in \u001b[36mDataArray.load\u001b[39m\u001b[34m(self, **kwargs)\u001b[39m\n\u001b[32m 1137\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mload\u001b[39m(\u001b[38;5;28mself\u001b[39m, **kwargs) -> Self:\n\u001b[32m 1138\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"Trigger loading data into memory and return this dataarray.\u001b[39;00m\n\u001b[32m 1139\u001b[39m \n\u001b[32m 1140\u001b[39m \u001b[33;03m Data will be computed and/or loaded from disk or a remote source.\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 1165\u001b[39m \u001b[33;03m Variable.load\u001b[39;00m\n\u001b[32m 1166\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m1167\u001b[39m ds = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_to_temp_dataset\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1168\u001b[39m new = \u001b[38;5;28mself\u001b[39m._from_temp_dataset(ds)\n\u001b[32m 1169\u001b[39m \u001b[38;5;28mself\u001b[39m._variable = new._variable\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/dev/work/azard/hacking-around/poc-climate-data/notebooks/icechunk-virtualizarr-minimal-reproducible-example-venv/lib/python3.12/site-packages/xarray/core/dataset.py:561\u001b[39m, in \u001b[36mDataset.load\u001b[39m\u001b[34m(self, **kwargs)\u001b[39m\n\u001b[32m 558\u001b[39m chunkmanager = get_chunked_array_type(*chunked_data.values())\n\u001b[32m 560\u001b[39m \u001b[38;5;66;03m# evaluate all the chunked arrays simultaneously\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m561\u001b[39m evaluated_data: \u001b[38;5;28mtuple\u001b[39m[np.ndarray[Any, Any], ...] = \u001b[43mchunkmanager\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcompute\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 562\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43mchunked_data\u001b[49m\u001b[43m.\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\n\u001b[32m 563\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 565\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m k, data \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(chunked_data, evaluated_data, strict=\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[32m 566\u001b[39m \u001b[38;5;28mself\u001b[39m.variables[k].data = data\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/dev/work/azard/hacking-around/poc-climate-data/notebooks/icechunk-virtualizarr-minimal-reproducible-example-venv/lib/python3.12/site-packages/xarray/namedarray/daskmanager.py:85\u001b[39m, in \u001b[36mDaskManager.compute\u001b[39m\u001b[34m(self, *data, **kwargs)\u001b[39m\n\u001b[32m 80\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mcompute\u001b[39m(\n\u001b[32m 81\u001b[39m \u001b[38;5;28mself\u001b[39m, *data: Any, **kwargs: Any\n\u001b[32m 82\u001b[39m ) -> \u001b[38;5;28mtuple\u001b[39m[np.ndarray[Any, _DType_co], ...]:\n\u001b[32m 83\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mdask\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01marray\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m compute\n\u001b[32m---> \u001b[39m\u001b[32m85\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mcompute\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/dev/work/azard/hacking-around/poc-climate-data/notebooks/icechunk-virtualizarr-minimal-reproducible-example-venv/lib/python3.12/site-packages/dask/base.py:681\u001b[39m, in \u001b[36mcompute\u001b[39m\u001b[34m(traverse, optimize_graph, scheduler, get, *args, **kwargs)\u001b[39m\n\u001b[32m 678\u001b[39m expr = expr.optimize()\n\u001b[32m 679\u001b[39m keys = \u001b[38;5;28mlist\u001b[39m(flatten(expr.__dask_keys__()))\n\u001b[32m--> \u001b[39m\u001b[32m681\u001b[39m results = \u001b[43mschedule\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexpr\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkeys\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 683\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m repack(results)\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/dev/work/azard/hacking-around/poc-climate-data/notebooks/icechunk-virtualizarr-minimal-reproducible-example-venv/lib/python3.12/site-packages/distributed/client.py:2417\u001b[39m, in \u001b[36mClient._gather\u001b[39m\u001b[34m(self, futures, errors, direct, local_worker)\u001b[39m\n\u001b[32m 2415\u001b[39m exception = st.exception\n\u001b[32m 2416\u001b[39m traceback = st.traceback\n\u001b[32m-> \u001b[39m\u001b[32m2417\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m exception.with_traceback(traceback)\n\u001b[32m 2418\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m errors == \u001b[33m\"\u001b[39m\u001b[33mskip\u001b[39m\u001b[33m\"\u001b[39m:\n\u001b[32m 2419\u001b[39m bad_keys.add(key)\n",
"\u001b[31mRuntimeError\u001b[39m: Error during deserialization of the task graph. This frequently\noccurs if the Scheduler and Client have different environments.\nFor more information, see\nhttps://docs.dask.org/en/stable/deployment-considerations.html#consistent-software-environments\n"
]
}
],
"source": [
"x = (ds.Tair > 0).sum()\n",
"x.load()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "54d2feeb-a20c-47ab-bdd6-b265b14fa521",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "icechunk-virtualizarr-minimal-reproducible-example",
"language": "python",
"name": "icechunk-virtualizarr-minimal-reproducible-example"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.7"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment