- python
- pip install requests
python fetch.py --organisation ministryofjustice --token ${token}
Last active
June 29, 2017 12:23
-
-
Save cliffxuan/3e2c302be3bd815844406d8c7437f1d9 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| # -*- coding: utf-8 -*- | |
| """ | |
| get the repositories of an organisation | |
| """ | |
| import os | |
| import re | |
| import json | |
| import urllib.parse | |
| import argparse | |
| import requests | |
| PWD = os.path.abspath(os.path.dirname(__file__)) | |
| BASE_URL = 'https://api.github.com' | |
| REGEX = ( | |
| '<(?P<next_url>.*)>; rel="next", ' | |
| '<(?P<last_url>.*?page=(?P<total>\d+))>; rel="last"' | |
| ) | |
| def get_total(token, organisation): | |
| rsp = requests.get( | |
| urllib.parse.urljoin(BASE_URL, 'orgs/{}/repos'.format(organisation)), | |
| headers={'Authorization': 'token {}'.format(token)}) | |
| link = rsp.headers['Link'] | |
| total = re.match(REGEX, link).groupdict()['total'] | |
| return int(total) | |
| def get_repos(token, organisation): | |
| repos = [] | |
| for page in range(1, get_total(token, organisation) + 1): | |
| rsp = requests.get( | |
| urllib.parse.urljoin( | |
| BASE_URL, | |
| 'orgs/ministryofjustice/repos?page={}'.format(page) | |
| ), | |
| headers={'Authorization': 'token {}'.format(token)} | |
| ) | |
| repos += rsp.json() | |
| return repos | |
| def argument_parser(): | |
| parser = argparse.ArgumentParser( | |
| description='dfetch repo data from github') | |
| parser.add_argument('--token', '-t', help='token', required=True) | |
| parser.add_argument( | |
| '--organisation', '-o', help='organisation', required=True) | |
| return parser | |
| def main(argv=None): | |
| args = argument_parser().parse_args(argv) | |
| print('fetch repo data from github') | |
| repos = get_repos(args.token, args.organisation) | |
| filename = os.path.join(PWD, 'repos.json') | |
| print('{} repositories found'.format(len(repos))) | |
| print('data saved to {}'.format(filename)) | |
| with open(filename, 'w') as fp: | |
| json.dump(repos, fp, indent=2) | |
| if __name__ == '__main__': | |
| main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment