Skip to content

Instantly share code, notes, and snippets.

@sharkinsspatial

sharkinsspatial/obstore.ipynb

Created November 11, 2024 19:01
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save sharkinsspatial/f3734b2d827e6c276043c729fd4326ac to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save sharkinsspatial/f3734b2d827e6c276043c729fd4326ac to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
obstore.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 5,
"id": "3f84e0e5-061a-44b5-a302-e0b4a02d5c13",
"metadata": {},
"outputs": [],
"source": [
"import boto3\n",
"import h5py\n",
"import obstore as obs\n",
"from obstore.store import S3Store"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "51363009-af03-444a-937d-8b988e519beb",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"S3Store(noaa-cdr-sea-surface-temp-optimum-interpolation-pds)"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"session = boto3.Session()\n",
"store = S3Store.from_session(session, \"noaa-cdr-sea-surface-temp-optimum-interpolation-pds\", config={\"AWS_REGION\": \"us-east-1\"})\n",
"store"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "26528cb9-3c26-42ad-bf6b-de45787982a8",
"metadata": {},
"outputs": [],
"source": [
"path = \"data/v2.1/avhrr/202401/oisst-avhrr-v02r01.20240105.nc\""
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "e5f6ad86-7546-4a4a-85ff-af4a356d3b9b",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<ReadableFile at 0x110ed9860>"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"file = obs.open(store, path)\n",
"file"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "b1f63b00-6810-452f-8d96-28c6e64bca6d",
"metadata": {},
"outputs": [
{
"ename": "TypeError",
"evalue": "ReadableFile.seek() missing 1 required positional argument: 'whence'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m h5_file \u001b[38;5;241m=\u001b[39m \u001b[43mh5py\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mFile\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2\u001b[0m h5_file\n",
"File \u001b[0;32m~/projects/obstore_experiments/venv/lib/python3.11/site-packages/h5py/_hl/files.py:561\u001b[0m, in \u001b[0;36mFile.__init__\u001b[0;34m(self, name, mode, driver, libver, userblock_size, swmr, rdcc_nslots, rdcc_nbytes, rdcc_w0, track_order, fs_strategy, fs_persist, fs_threshold, fs_page_size, page_buf_size, min_meta_keep, min_raw_keep, locking, alignment_threshold, alignment_interval, meta_block_size, **kwds)\u001b[0m\n\u001b[1;32m 552\u001b[0m fapl \u001b[38;5;241m=\u001b[39m make_fapl(driver, libver, rdcc_nslots, rdcc_nbytes, rdcc_w0,\n\u001b[1;32m 553\u001b[0m locking, page_buf_size, min_meta_keep, min_raw_keep,\n\u001b[1;32m 554\u001b[0m alignment_threshold\u001b[38;5;241m=\u001b[39malignment_threshold,\n\u001b[1;32m 555\u001b[0m alignment_interval\u001b[38;5;241m=\u001b[39malignment_interval,\n\u001b[1;32m 556\u001b[0m meta_block_size\u001b[38;5;241m=\u001b[39mmeta_block_size,\n\u001b[1;32m 557\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwds)\n\u001b[1;32m 558\u001b[0m fcpl \u001b[38;5;241m=\u001b[39m make_fcpl(track_order\u001b[38;5;241m=\u001b[39mtrack_order, fs_strategy\u001b[38;5;241m=\u001b[39mfs_strategy,\n\u001b[1;32m 559\u001b[0m fs_persist\u001b[38;5;241m=\u001b[39mfs_persist, fs_threshold\u001b[38;5;241m=\u001b[39mfs_threshold,\n\u001b[1;32m 560\u001b[0m fs_page_size\u001b[38;5;241m=\u001b[39mfs_page_size)\n\u001b[0;32m--> 561\u001b[0m fid \u001b[38;5;241m=\u001b[39m \u001b[43mmake_fid\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43muserblock_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfapl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfcpl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mswmr\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mswmr\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 563\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(libver, \u001b[38;5;28mtuple\u001b[39m):\n\u001b[1;32m 564\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_libver \u001b[38;5;241m=\u001b[39m libver\n",
"File \u001b[0;32m~/projects/obstore_experiments/venv/lib/python3.11/site-packages/h5py/_hl/files.py:235\u001b[0m, in \u001b[0;36mmake_fid\u001b[0;34m(name, mode, userblock_size, fapl, fcpl, swmr)\u001b[0m\n\u001b[1;32m 233\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m swmr \u001b[38;5;129;01mand\u001b[39;00m swmr_support:\n\u001b[1;32m 234\u001b[0m flags \u001b[38;5;241m|\u001b[39m\u001b[38;5;241m=\u001b[39m h5f\u001b[38;5;241m.\u001b[39mACC_SWMR_READ\n\u001b[0;32m--> 235\u001b[0m fid \u001b[38;5;241m=\u001b[39m \u001b[43mh5f\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mflags\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfapl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfapl\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 236\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m mode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mr+\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m 237\u001b[0m fid \u001b[38;5;241m=\u001b[39m h5f\u001b[38;5;241m.\u001b[39mopen(name, h5f\u001b[38;5;241m.\u001b[39mACC_RDWR, fapl\u001b[38;5;241m=\u001b[39mfapl)\n",
"File \u001b[0;32mh5py/_objects.pyx:54\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32mh5py/_objects.pyx:55\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32mh5py/h5f.pyx:102\u001b[0m, in \u001b[0;36mh5py.h5f.open\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32mh5py/h5fd.pyx:159\u001b[0m, in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;31mTypeError\u001b[0m: ReadableFile.seek() missing 1 required positional argument: 'whence'"
]
}
],
"source": [
"h5_file = h5py.File(file, \"r\")\n",
"h5_file"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f47b4bbd-fb53-4c44-b4ed-95ae8db08477",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.0"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment