Created
February 14, 2023 09:08
-
-
Save NatLee/994d44ac31e3e9525a0a6ec3a75ad870 to your computer and use it in GitHub Desktop.
A python wrapped utility for making a backup of repo in self-host Gitlab
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from concurrent.futures import ThreadPoolExecutor, wait | |
| import subprocess | |
| from pathlib import Path | |
| from loguru import logger | |
| import requests | |
| HOST = '<YOUR_GITLAB_HOST>' | |
| TOKEN = '<YOUR_TOKEN_TO_READ_REPO>' | |
| GITLABUSERNAME = '<YOUR_GITLAB_USERNAME>' | |
| NUMBER_OF_PROJECTS = 100 | |
| BACKUP_FOLDER = './gitlab-backup' | |
| def system_call(command): | |
| return subprocess.check_output(command, shell=True, close_fds=True) | |
| def git_clone(url): | |
| return system_call(f'cd {BACKUP_FOLDER} && git clone {url}') | |
| def clone_all_project(): | |
| req = requests.get(f'{HOST}/api/v4/projects?private_token={TOKEN}&per_page={NUMBER_OF_PROJECTS}') | |
| git_urls = [p['http_url_to_repo'] for p in req.json()] | |
| with ThreadPoolExecutor(max_workers=4) as executor: | |
| jobs = [] | |
| for url in git_urls: | |
| jobs.append( | |
| executor.submit(git_clone, url) | |
| ) | |
| wait(jobs) | |
| for job in jobs: | |
| print(job.result()) | |
| def create_project(project_name:str, auto_devops:bool=False): | |
| return system_call(f'curl --header "PRIVATE-TOKEN: {TOKEN}" -X POST "{HOST}/api/v4/projects?name={project_name}&auto_devops_enabled={str(auto_devops).lower()}"') | |
| def push_old_project_to_new_project(project_name:str): | |
| return system_call(f'cd {BACKUP_FOLDER}/{project_name} && \ | |
| git remote rename origin old-origin && \ | |
| git remote add origin {HOST}/{GITLABUSERNAME}/{project_name}.git && \ | |
| git push -u origin --all && \ | |
| git push -u origin --tags' | |
| ) | |
| def create_and_push(project_name:str): | |
| logger.info(project_name) | |
| create_project(project_name) | |
| push_old_project_to_new_project(project_name) | |
| return project_name | |
| def push_all_old_projects(projects_folder:str, threaded=True): | |
| folders = [folder.name for folder in Path(projects_folder).glob('*') if not folder.name.startswith('.')] | |
| if threaded: | |
| with ThreadPoolExecutor(max_workers=4) as executor: | |
| jobs = [] | |
| for folder in folders: | |
| jobs.append( | |
| executor.submit(create_and_push, folder) | |
| ) | |
| wait(jobs) | |
| else: | |
| for folder in folders: | |
| try: | |
| create_and_push(folder) | |
| except Exception as e: | |
| import pdb; pdb.set_trace() | |
| return |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| requests | |
| loguru |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment