Last active
December 20, 2024 12:12
-
-
Save shacharmirkin/cb0fa7662dcab51462ca9c449009efb3 to your computer and use it in GitHub Desktop.
using multiple virtual environments in Google Colab
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "KUmLRpHOMOMm" | |
| }, | |
| "source": [ | |
| "# Multiple venvs in Google Colab\n", | |
| "\n", | |
| "This notebook demonstrates how to use multiple virtual environments in Google Colab, e.g. when you need to work with 2 different versions of a package.\n", | |
| "\n", | |
| "[](https://colab.research.google.com/gist/shacharmirkin/cb0fa7662dcab51462ca9c449009efb3#file-multiple_venvs_in_colab-ipynb)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "W8MmKq4lYgfb" | |
| }, | |
| "source": [ | |
| "Creating the virtual environments.\n", | |
| "The only difference in this example is the yaml version." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "0joppeW6k2PE" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "import os\n", | |
| "import subprocess\n", | |
| "import sys\n", | |
| "import sysconfig" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "vgf_AiTZFfVz" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Paths for virtual environments. /content is a temporary folder; the virtualenv can be created elsewhere\n", | |
| "ENV1 = \"/content/env1\"\n", | |
| "ENV2 = \"/content/env2\"\n", | |
| "\n", | |
| "def create_env(env_path, packages):\n", | |
| " \"\"\"Create a virtual environment using virtualenv.\"\"\"\n", | |
| " if not os.path.exists(env_path):\n", | |
| " print(f\"Creating virtual environment at {env_path}...\")\n", | |
| " subprocess.run([\"pip\", \"install\", \"--upgrade\", \"virtualenv\"], check=True)\n", | |
| " subprocess.run([\"python3\", \"-m\", \"virtualenv\", env_path], check=True)\n", | |
| " subprocess.run([f\"{env_path}/bin/pip\", \"install\", \"--upgrade\", \"pip\"], check=True)\n", | |
| " subprocess.run([f\"{env_path}/bin/pip\", \"install\"] + packages, check=True)\n", | |
| " print(f\"Virtual environment at {env_path} created successfully.\")\n", | |
| "\n", | |
| "ENV1_REQS = [\"PyYAML==6.0.2\"]\n", | |
| "ENV2_REQS = [\"PyYAML==6.0.1\"]\n", | |
| "\n", | |
| "create_env(ENV1, ENV1_REQS)\n", | |
| "create_env(ENV2, ENV2_REQS)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "Yqyb8kqHYpB4" | |
| }, | |
| "source": [ | |
| "Switching between environments" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "WOVJ888uYIZ0" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "def switch_to_env(env_path: str, to_unload: list[str]):\n", | |
| " \"\"\"Switch to a virtual environment and unload specific packages\"\"\"\n", | |
| " for package in to_unload:\n", | |
| " if package in sys.modules:\n", | |
| " del sys.modules[package]\n", | |
| "\n", | |
| " python_version = f\"python{sysconfig.get_python_version().split('.')[0]}.{sysconfig.get_python_version().split('.')[1]}\"\n", | |
| " site_packages_path = os.path.join(env_path, 'lib', python_version, 'site-packages')\n", | |
| " if os.path.exists(site_packages_path):\n", | |
| " sys.path.insert(0, site_packages_path)\n", | |
| " print(f\"Switched to environment: {env_path}\")\n", | |
| " else:\n", | |
| " print(f\"Error: Virtual environment at {env_path} not found.\")\n", | |
| " return\n" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "EhqhAQKSjQIC" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "switch_to_env(ENV1, [\"yaml\"])\n", | |
| "import yaml\n", | |
| "yaml.__version__" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "p-GhQqRiktkB" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "switch_to_env(ENV2, [\"yaml\"])\n", | |
| "import yaml\n", | |
| "yaml.__version__" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "BQ7GI3vhYPCQ" | |
| }, | |
| "source": [ | |
| "Alternatively, run in the virtual env without switching into it. \n", | |
| "This is more coherent with respect to the loaded packages, but less convenient to use." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "CkfEBy6lF-SZ" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "def run_in_env(env_path: str, code_to_execute: str):\n", | |
| " \"\"\"Run a Python code snippet in the specified virtual environment\"\"\"\n", | |
| " python_exec = os.path.join(env_path, \"bin\", \"python\")\n", | |
| " result = subprocess.run([python_exec, \"-c\", code_to_execute], capture_output=True, text=True)\n", | |
| " print(f\"{env_path}:\", result.stdout)\n", | |
| "\n", | |
| "env1_code = \"import yaml; print(yaml.__version__)\"\n", | |
| "run_in_env(ENV1, env1_code)\n", | |
| "\n", | |
| "env2_code = \"import yaml; print(yaml.__version__)\"\n", | |
| "run_in_env(ENV2, env2_code)" | |
| ] | |
| } | |
| ], | |
| "metadata": { | |
| "colab": { | |
| "private_outputs": true, | |
| "provenance": [] | |
| }, | |
| "kernelspec": { | |
| "display_name": "Python 3", | |
| "name": "python3" | |
| }, | |
| "language_info": { | |
| "name": "python" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 0 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment