Last active
February 28, 2024 19:01
-
-
Save alimanfoo/eea0fdec41333a8c39d04f72a89dfd9b to your computer and use it in GitHub Desktop.
How to create links with zarr
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "Examples of creating links using Zarr, addressing comments in [GH297](https://github.com/zarr-developers/zarr/issues/297)." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 1, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "mkdir: created directory 'data'\r\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "import zarr\n", | |
| "!mkdir -pv data" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "An automated process dumps data every day. They use Zarr and create a new root group for the data from each data. E.g.:" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 2, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "# day 1\n", | |
| "root = zarr.open('data/daily/2018-01-01')\n", | |
| "# ... save some data.\n", | |
| "# day 2\n", | |
| "root = zarr.open('data/daily/2018-01-02')\n", | |
| "# ... save some data.\n", | |
| "# day 3\n", | |
| "root = zarr.open('data/daily/2018-01-03')\n", | |
| "# ... save some data.\n", | |
| "# etc." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 3, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "\u001b[01;34mdata\u001b[00m\r\n", | |
| "└── \u001b[01;34mdaily\u001b[00m\r\n", | |
| " ├── \u001b[01;34m2018-01-01\u001b[00m\r\n", | |
| " │ └── .zgroup\r\n", | |
| " ├── \u001b[01;34m2018-01-02\u001b[00m\r\n", | |
| " │ └── .zgroup\r\n", | |
| " └── \u001b[01;34m2018-01-03\u001b[00m\r\n", | |
| " └── .zgroup\r\n", | |
| "\r\n", | |
| "4 directories, 3 files\r\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "!tree -a data" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "# View as single dataset" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "A user now wants to view this as one big dataset, rather than separate datasets." | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "## Option 1 - turn the parent directory into a group\n", | |
| "\n", | |
| "In Zarr there is nothing special about a root group. You can turn any directory into a group, and a directory that was initially created as a root group can become a child group. E.g.:" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 4, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<link rel=\"stylesheet\" href=\"//cdnjs.cloudflare.com/ajax/libs/jstree/3.3.3/themes/default/style.min.css\"/><div id=\"12e8336e-ecfa-45fc-9312-70a392528517\" class=\"zarr-tree\"><ul><li data-jstree='{\"type\": \"Group\"}' class='jstree-open'><span>/</span><ul><li data-jstree='{\"type\": \"Group\"}' class=''><span>2018-01-01</span></li><li data-jstree='{\"type\": \"Group\"}' class=''><span>2018-01-02</span></li><li data-jstree='{\"type\": \"Group\"}' class=''><span>2018-01-03</span></li></ul></li></ul></div>\n", | |
| "<script>\n", | |
| " if (!require.defined('jquery')) {\n", | |
| " require.config({\n", | |
| " paths: {\n", | |
| " jquery: '//cdnjs.cloudflare.com/ajax/libs/jquery/1.12.1/jquery.min'\n", | |
| " },\n", | |
| " });\n", | |
| " }\n", | |
| " if (!require.defined('jstree')) {\n", | |
| " require.config({\n", | |
| " paths: {\n", | |
| " jstree: '//cdnjs.cloudflare.com/ajax/libs/jstree/3.3.3/jstree.min'\n", | |
| " },\n", | |
| " });\n", | |
| " }\n", | |
| " require(['jstree'], function() {\n", | |
| " $('#12e8336e-ecfa-45fc-9312-70a392528517').jstree({\n", | |
| " types: {\n", | |
| " Group: {\n", | |
| " icon: \"fa fa-folder\"\n", | |
| " },\n", | |
| " Array: {\n", | |
| " icon: \"fa fa-table\"\n", | |
| " }\n", | |
| " },\n", | |
| " plugins: [\"types\"]\n", | |
| " });\n", | |
| " });\n", | |
| "</script>\n" | |
| ], | |
| "text/plain": [ | |
| "/\n", | |
| " ├── 2018-01-01\n", | |
| " ├── 2018-01-02\n", | |
| " └── 2018-01-03" | |
| ] | |
| }, | |
| "execution_count": 4, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "daily = zarr.open('data/daily')\n", | |
| "daily.tree()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 5, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": [ | |
| "<zarr.hierarchy.Group '/2018-01-03'>" | |
| ] | |
| }, | |
| "execution_count": 5, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "daily['2018-01-03']" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 6, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "\u001b[01;34mdata\u001b[00m\r\n", | |
| "└── \u001b[01;34mdaily\u001b[00m\r\n", | |
| " ├── \u001b[01;34m2018-01-01\u001b[00m\r\n", | |
| " │ └── .zgroup\r\n", | |
| " ├── \u001b[01;34m2018-01-02\u001b[00m\r\n", | |
| " │ └── .zgroup\r\n", | |
| " ├── \u001b[01;34m2018-01-03\u001b[00m\r\n", | |
| " │ └── .zgroup\r\n", | |
| " └── .zgroup\r\n", | |
| "\r\n", | |
| "4 directories, 4 files\r\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "!tree -a data" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "Note that a new '.zgroup' file has been created within the \"daily\" directory." | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "## Option 2 - use file system links\n", | |
| "\n", | |
| "This approach is more flexible, as you can create any kind of view you want. E.g.:" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 7, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "mkdir: created directory 'data/all'\n", | |
| "'data/daily/2018-01-01' -> 'data/all/2018-01-01'\n", | |
| "'data/daily/2018-01-01/.zgroup' -> 'data/all/2018-01-01/.zgroup'\n", | |
| "'data/daily/2018-01-02' -> 'data/all/2018-01-02'\n", | |
| "'data/daily/2018-01-02/.zgroup' -> 'data/all/2018-01-02/.zgroup'\n", | |
| "'data/daily/2018-01-03' -> 'data/all/2018-01-03'\n", | |
| "'data/daily/2018-01-03/.zgroup' -> 'data/all/2018-01-03/.zgroup'\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "!mkdir -pv data/all\n", | |
| "!cp -rv --link data/daily/* data/all" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 8, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<link rel=\"stylesheet\" href=\"//cdnjs.cloudflare.com/ajax/libs/jstree/3.3.3/themes/default/style.min.css\"/><div id=\"123d4430-5e8e-48db-b449-b9466a1306d0\" class=\"zarr-tree\"><ul><li data-jstree='{\"type\": \"Group\"}' class='jstree-open'><span>/</span><ul><li data-jstree='{\"type\": \"Group\"}' class=''><span>2018-01-01</span></li><li data-jstree='{\"type\": \"Group\"}' class=''><span>2018-01-02</span></li><li data-jstree='{\"type\": \"Group\"}' class=''><span>2018-01-03</span></li></ul></li></ul></div>\n", | |
| "<script>\n", | |
| " if (!require.defined('jquery')) {\n", | |
| " require.config({\n", | |
| " paths: {\n", | |
| " jquery: '//cdnjs.cloudflare.com/ajax/libs/jquery/1.12.1/jquery.min'\n", | |
| " },\n", | |
| " });\n", | |
| " }\n", | |
| " if (!require.defined('jstree')) {\n", | |
| " require.config({\n", | |
| " paths: {\n", | |
| " jstree: '//cdnjs.cloudflare.com/ajax/libs/jstree/3.3.3/jstree.min'\n", | |
| " },\n", | |
| " });\n", | |
| " }\n", | |
| " require(['jstree'], function() {\n", | |
| " $('#123d4430-5e8e-48db-b449-b9466a1306d0').jstree({\n", | |
| " types: {\n", | |
| " Group: {\n", | |
| " icon: \"fa fa-folder\"\n", | |
| " },\n", | |
| " Array: {\n", | |
| " icon: \"fa fa-table\"\n", | |
| " }\n", | |
| " },\n", | |
| " plugins: [\"types\"]\n", | |
| " });\n", | |
| " });\n", | |
| "</script>\n" | |
| ], | |
| "text/plain": [ | |
| "/\n", | |
| " ├── 2018-01-01\n", | |
| " ├── 2018-01-02\n", | |
| " └── 2018-01-03" | |
| ] | |
| }, | |
| "execution_count": 8, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "data = zarr.open('data/all')\n", | |
| "data.tree()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 9, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "\u001b[01;34mdata/all\u001b[00m\r\n", | |
| "├── \u001b[01;34m2018-01-01\u001b[00m\r\n", | |
| "│ └── .zgroup\r\n", | |
| "├── \u001b[01;34m2018-01-02\u001b[00m\r\n", | |
| "│ └── .zgroup\r\n", | |
| "├── \u001b[01;34m2018-01-03\u001b[00m\r\n", | |
| "│ └── .zgroup\r\n", | |
| "└── .zgroup\r\n", | |
| "\r\n", | |
| "3 directories, 4 files\r\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "!tree -a data/all" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "# View latest day of data\n", | |
| "\n", | |
| "Could be done with file system links. E.g.:" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 10, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "'data/daily/2018-01-01' -> 'data/daily/latest'\r\n", | |
| "'data/daily/2018-01-01/.zgroup' -> 'data/daily/latest/.zgroup'\r\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "!cp -rv --link data/daily/2018-01-01 data/daily/latest" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 11, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<link rel=\"stylesheet\" href=\"//cdnjs.cloudflare.com/ajax/libs/jstree/3.3.3/themes/default/style.min.css\"/><div id=\"3ba59d64-f93c-414a-9482-419440ce857f\" class=\"zarr-tree\"><ul><li data-jstree='{\"type\": \"Group\"}' class='jstree-open'><span>/</span><ul><li data-jstree='{\"type\": \"Group\"}' class=''><span>2018-01-01</span></li><li data-jstree='{\"type\": \"Group\"}' class=''><span>2018-01-02</span></li><li data-jstree='{\"type\": \"Group\"}' class=''><span>2018-01-03</span></li><li data-jstree='{\"type\": \"Group\"}' class=''><span>latest</span></li></ul></li></ul></div>\n", | |
| "<script>\n", | |
| " if (!require.defined('jquery')) {\n", | |
| " require.config({\n", | |
| " paths: {\n", | |
| " jquery: '//cdnjs.cloudflare.com/ajax/libs/jquery/1.12.1/jquery.min'\n", | |
| " },\n", | |
| " });\n", | |
| " }\n", | |
| " if (!require.defined('jstree')) {\n", | |
| " require.config({\n", | |
| " paths: {\n", | |
| " jstree: '//cdnjs.cloudflare.com/ajax/libs/jstree/3.3.3/jstree.min'\n", | |
| " },\n", | |
| " });\n", | |
| " }\n", | |
| " require(['jstree'], function() {\n", | |
| " $('#3ba59d64-f93c-414a-9482-419440ce857f').jstree({\n", | |
| " types: {\n", | |
| " Group: {\n", | |
| " icon: \"fa fa-folder\"\n", | |
| " },\n", | |
| " Array: {\n", | |
| " icon: \"fa fa-table\"\n", | |
| " }\n", | |
| " },\n", | |
| " plugins: [\"types\"]\n", | |
| " });\n", | |
| " });\n", | |
| "</script>\n" | |
| ], | |
| "text/plain": [ | |
| "/\n", | |
| " ├── 2018-01-01\n", | |
| " ├── 2018-01-02\n", | |
| " ├── 2018-01-03\n", | |
| " └── latest" | |
| ] | |
| }, | |
| "execution_count": 11, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "daily = zarr.open('data/daily')\n", | |
| "daily.tree()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 12, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "removed 'data/daily/latest/.zgroup'\n", | |
| "removed directory 'data/daily/latest'\n", | |
| "'data/daily/2018-01-02' -> 'data/daily/latest'\n", | |
| "'data/daily/2018-01-02/.zgroup' -> 'data/daily/latest/.zgroup'\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "# update latest link\n", | |
| "!rm -rv data/daily/latest\n", | |
| "!cp -rv --link data/daily/2018-01-02 data/daily/latest" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "# More examples\n", | |
| "\n", | |
| "Here's a possible solution to a similar requirement, expressed slightly differently, by @onalant:\n", | |
| "\n", | |
| ">Here is my understanding of the problem. Take some zarr store, a.zarr.\n", | |
| "Every day, some application writes some data to a.zarr. However, it\n", | |
| "groups the data together by the date on which it was written. We may\n", | |
| "have have groups like /2018/08/30, for example. What @mrocklin seems\n", | |
| "to be proposing is having multiple metadata files that \"transmute\" the\n", | |
| "user-facing appearance of a.zarr. Suppose we also had b.zarr and\n", | |
| "c.zarr, two stores that refer to a.zarr for data. However, b.zarr\n", | |
| "specifies in its metadata that it shows the \"latest\" data entries\n", | |
| "(/2018/08/30, e.g.), while c.zarr \"flattens\" all of the data in\n", | |
| "a.zarr to appear as though everything were under the root group.\n", | |
| "\n" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 13, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": [ | |
| "<zarr.hierarchy.Group '/2018/09/01'>" | |
| ] | |
| }, | |
| "execution_count": 13, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "a = zarr.open('data/a.zarr')\n", | |
| "# day 1\n", | |
| "a.create_group('2018/08/30')\n", | |
| "# day 2\n", | |
| "a.create_group('2018/08/31')\n", | |
| "# day 3\n", | |
| "a.create_group('2018/09/01')\n", | |
| "# etc." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 14, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<link rel=\"stylesheet\" href=\"//cdnjs.cloudflare.com/ajax/libs/jstree/3.3.3/themes/default/style.min.css\"/><div id=\"8b291562-d4f0-482e-af3f-a08445395ba7\" class=\"zarr-tree\"><ul><li data-jstree='{\"type\": \"Group\"}' class='jstree-open'><span>/</span><ul><li data-jstree='{\"type\": \"Group\"}' class='jstree-open'><span>2018</span><ul><li data-jstree='{\"type\": \"Group\"}' class='jstree-open'><span>08</span><ul><li data-jstree='{\"type\": \"Group\"}' class='jstree-open'><span>30</span></li><li data-jstree='{\"type\": \"Group\"}' class='jstree-open'><span>31</span></li></ul></li><li data-jstree='{\"type\": \"Group\"}' class='jstree-open'><span>09</span><ul><li data-jstree='{\"type\": \"Group\"}' class='jstree-open'><span>01</span></li></ul></li></ul></li></ul></li></ul></div>\n", | |
| "<script>\n", | |
| " if (!require.defined('jquery')) {\n", | |
| " require.config({\n", | |
| " paths: {\n", | |
| " jquery: '//cdnjs.cloudflare.com/ajax/libs/jquery/1.12.1/jquery.min'\n", | |
| " },\n", | |
| " });\n", | |
| " }\n", | |
| " if (!require.defined('jstree')) {\n", | |
| " require.config({\n", | |
| " paths: {\n", | |
| " jstree: '//cdnjs.cloudflare.com/ajax/libs/jstree/3.3.3/jstree.min'\n", | |
| " },\n", | |
| " });\n", | |
| " }\n", | |
| " require(['jstree'], function() {\n", | |
| " $('#8b291562-d4f0-482e-af3f-a08445395ba7').jstree({\n", | |
| " types: {\n", | |
| " Group: {\n", | |
| " icon: \"fa fa-folder\"\n", | |
| " },\n", | |
| " Array: {\n", | |
| " icon: \"fa fa-table\"\n", | |
| " }\n", | |
| " },\n", | |
| " plugins: [\"types\"]\n", | |
| " });\n", | |
| " });\n", | |
| "</script>\n" | |
| ], | |
| "text/plain": [ | |
| "/\n", | |
| " └── 2018\n", | |
| " ├── 08\n", | |
| " │ ├── 30\n", | |
| " │ └── 31\n", | |
| " └── 09\n", | |
| " └── 01" | |
| ] | |
| }, | |
| "execution_count": 14, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "a.tree(expand=True)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 15, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "mkdir: created directory 'data/b.zarr'\n", | |
| "'data/a.zarr/2018/09/01' -> 'data/b.zarr/latest'\n", | |
| "'data/a.zarr/2018/09/01/.zgroup' -> 'data/b.zarr/latest/.zgroup'\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "# create a \"latest\" dataset\n", | |
| "!mkdir -pv data/b.zarr\n", | |
| "!cp -rv --link data/a.zarr/2018/09/01 data/b.zarr/latest" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 16, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<link rel=\"stylesheet\" href=\"//cdnjs.cloudflare.com/ajax/libs/jstree/3.3.3/themes/default/style.min.css\"/><div id=\"cb8ff541-f6d3-4f34-bf5f-e5b5e7b444e1\" class=\"zarr-tree\"><ul><li data-jstree='{\"type\": \"Group\"}' class='jstree-open'><span>/</span><ul><li data-jstree='{\"type\": \"Group\"}' class=''><span>latest</span></li></ul></li></ul></div>\n", | |
| "<script>\n", | |
| " if (!require.defined('jquery')) {\n", | |
| " require.config({\n", | |
| " paths: {\n", | |
| " jquery: '//cdnjs.cloudflare.com/ajax/libs/jquery/1.12.1/jquery.min'\n", | |
| " },\n", | |
| " });\n", | |
| " }\n", | |
| " if (!require.defined('jstree')) {\n", | |
| " require.config({\n", | |
| " paths: {\n", | |
| " jstree: '//cdnjs.cloudflare.com/ajax/libs/jstree/3.3.3/jstree.min'\n", | |
| " },\n", | |
| " });\n", | |
| " }\n", | |
| " require(['jstree'], function() {\n", | |
| " $('#cb8ff541-f6d3-4f34-bf5f-e5b5e7b444e1').jstree({\n", | |
| " types: {\n", | |
| " Group: {\n", | |
| " icon: \"fa fa-folder\"\n", | |
| " },\n", | |
| " Array: {\n", | |
| " icon: \"fa fa-table\"\n", | |
| " }\n", | |
| " },\n", | |
| " plugins: [\"types\"]\n", | |
| " });\n", | |
| " });\n", | |
| "</script>\n" | |
| ], | |
| "text/plain": [ | |
| "/\n", | |
| " └── latest" | |
| ] | |
| }, | |
| "execution_count": 16, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "b = zarr.open('data/b.zarr')\n", | |
| "b.tree()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 17, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "mkdir: created directory 'data/c.zarr'\n", | |
| "year: 2018\n", | |
| "month: 08\n", | |
| "'data/a.zarr/2018/08/30' -> 'data/c.zarr/2018-08-30'\n", | |
| "'data/a.zarr/2018/08/30/.zgroup' -> 'data/c.zarr/2018-08-30/.zgroup'\n", | |
| "'data/a.zarr/2018/08/31' -> 'data/c.zarr/2018-08-31'\n", | |
| "'data/a.zarr/2018/08/31/.zgroup' -> 'data/c.zarr/2018-08-31/.zgroup'\n", | |
| "month: 09\n", | |
| "'data/a.zarr/2018/09/01' -> 'data/c.zarr/2018-09-01'\n", | |
| "'data/a.zarr/2018/09/01/.zgroup' -> 'data/c.zarr/2018-09-01/.zgroup'\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "%%bash\n", | |
| "# create a flattened hierarchy\n", | |
| "mkdir -pv data/c.zarr\n", | |
| "for year in $(ls -1 data/a.zarr); do\n", | |
| " echo year: $year\n", | |
| " for month in $(ls -1 data/a.zarr/${year}); do\n", | |
| " echo month: $month\n", | |
| " for day in $(ls -1 data/a.zarr/${year}/${month}); do\n", | |
| " cp -rv --link data/a.zarr/${year}/${month}/${day} data/c.zarr/${year}-${month}-${day}\n", | |
| " done\n", | |
| " done\n", | |
| "done\n" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 18, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<link rel=\"stylesheet\" href=\"//cdnjs.cloudflare.com/ajax/libs/jstree/3.3.3/themes/default/style.min.css\"/><div id=\"bbff88d5-8c1b-44e3-b83a-dce582f3c9ea\" class=\"zarr-tree\"><ul><li data-jstree='{\"type\": \"Group\"}' class='jstree-open'><span>/</span><ul><li data-jstree='{\"type\": \"Group\"}' class=''><span>2018-08-30</span></li><li data-jstree='{\"type\": \"Group\"}' class=''><span>2018-08-31</span></li><li data-jstree='{\"type\": \"Group\"}' class=''><span>2018-09-01</span></li></ul></li></ul></div>\n", | |
| "<script>\n", | |
| " if (!require.defined('jquery')) {\n", | |
| " require.config({\n", | |
| " paths: {\n", | |
| " jquery: '//cdnjs.cloudflare.com/ajax/libs/jquery/1.12.1/jquery.min'\n", | |
| " },\n", | |
| " });\n", | |
| " }\n", | |
| " if (!require.defined('jstree')) {\n", | |
| " require.config({\n", | |
| " paths: {\n", | |
| " jstree: '//cdnjs.cloudflare.com/ajax/libs/jstree/3.3.3/jstree.min'\n", | |
| " },\n", | |
| " });\n", | |
| " }\n", | |
| " require(['jstree'], function() {\n", | |
| " $('#bbff88d5-8c1b-44e3-b83a-dce582f3c9ea').jstree({\n", | |
| " types: {\n", | |
| " Group: {\n", | |
| " icon: \"fa fa-folder\"\n", | |
| " },\n", | |
| " Array: {\n", | |
| " icon: \"fa fa-table\"\n", | |
| " }\n", | |
| " },\n", | |
| " plugins: [\"types\"]\n", | |
| " });\n", | |
| " });\n", | |
| "</script>\n" | |
| ], | |
| "text/plain": [ | |
| "/\n", | |
| " ├── 2018-08-30\n", | |
| " ├── 2018-08-31\n", | |
| " └── 2018-09-01" | |
| ] | |
| }, | |
| "execution_count": 18, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "c = zarr.open('data/c.zarr')\n", | |
| "c.tree()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [] | |
| } | |
| ], | |
| "metadata": { | |
| "kernelspec": { | |
| "display_name": "Python 3", | |
| "language": "python", | |
| "name": "python3" | |
| }, | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython3", | |
| "version": "3.6.5" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 2 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment