Skip to content

Instantly share code, notes, and snippets.

@nikhilkumarsingh

nikhilkumarsingh/concurrent_task_progress_bar.ipynb

Last active August 29, 2024 02:57
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save nikhilkumarsingh/be174036a47d50ca4a7a9c48885f534c to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save nikhilkumarsingh/be174036a47d50ca4a7a9c48885f534c to your computer and use it in GitHub Desktop.
Download ZIP
Displaying Progress Bar for Concurrent Tasks
Display the source blob
Display the rendered blob
Raw
concurrent_task_progress_bar.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Displaying Progress Bar for Concurrent Tasks"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"`pip install tqdm`"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import time\n",
"from tqdm.notebook import tqdm"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"N = 30"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"def foo(a):\n",
" time.sleep(0.2)\n",
" return a**2"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "244c953a957a4a76a25013b4c95a8779",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=30.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"x = []\n",
"for i in tqdm(range(N)):\n",
" x.append(foo(i))"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "440b6330404a481f894b350bfe2a8d51",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=30.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"x = [foo(i) for i in tqdm(range(N))]"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "0c1f3ddb25184b0a90318fdee4aa6d51",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=30.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"x = list(tqdm(map(foo, range(N)), total=N))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## multiprocessing"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"from multiprocessing.pool import Pool\n",
"from concurrent.futures import ProcessPoolExecutor\n",
"from tqdm.contrib.concurrent import process_map"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "a9fadfb8d64d43e1a10b9ac1aa53b075",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=30.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"with Pool() as p:\n",
" x = list(tqdm(p.imap(foo, range(N)), total=N))"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "9f7f21ce1ace415bb4f22a23d42420ee",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=30.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"with ProcessPoolExecutor() as executor:\n",
" x = list(tqdm(executor.map(foo, range(N)), total=N))"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "771ee2ccf73341729607cef5fcac8af9",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=30.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"x = process_map(foo, range(N))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## multithreading"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"from multiprocessing.pool import ThreadPool\n",
"from concurrent.futures import ThreadPoolExecutor\n",
"from tqdm.contrib.concurrent import thread_map"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "9e312c7a2e9b44e0a411f26681945d19",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=30.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"with ThreadPool(processes=10) as p:\n",
" x = list(tqdm(p.imap(foo, range(N)), total=N))"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "1db27a276cc64b478003d767b70edb4c",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=30.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"with ThreadPoolExecutor(max_workers=10) as executor:\n",
" x = list(tqdm(executor.map(foo, range(N)), total=N))"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "67acf598d5304074934ad4d686649be4",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=30.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"x = thread_map(foo, range(N), max_workers=10)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## asyncio"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"`pip install asyncio`\n",
"\n",
"`pip install nest_asyncio`"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"import asyncio\n",
"import nest_asyncio\n",
"nest_asyncio.apply()"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"async def afoo(a):\n",
" await asyncio.sleep(0.2)\n",
" return a**2"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"4"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"asyncio.run(afoo(2))"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"tasks = list(map(afoo, range(N)))\n",
"x = asyncio.run(asyncio.gather(*tasks))"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"async def do1():\n",
" tasks = list(map(afoo, range(N)))\n",
" return [await t for t in tqdm(asyncio.as_completed(tasks), total=N)]"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "a3826e4382aa48be854c718d3e97f8a6",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=30.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"x = asyncio.run(do1())"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"async def do2():\n",
" tasks = list(map(asyncio.create_task, map(afoo, range(N))))\n",
" for t in tqdm(asyncio.as_completed(tasks), total=N):\n",
" await t\n",
" return [t.result() for t in tasks]"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "824f42244421414ea4d0b3961db345d0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=30.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"x = asyncio.run(do2())"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "pyenv37",
"language": "python",
"name": "pyenv37"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
@casperdcl
Copy link

casperdcl commented Nov 6, 2020

if you use the auto version:

-from tqdm.notebook import tqdm
+from tqdm.auto import tqdm

you can simplify this:

-tqdm(asyncio.as_completed(tasks), total=N)
+tqdm.as_completed(tasks)

You also may as well use a list comprehension in do2() as in do1():

-    for t in tqdm(asyncio.as_completed(tasks), total=N):
-        await t
+    [await t for t in tqdm.as_completed(tasks)]

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment