github-backup

#!/usr/bin/env python

from __future__ import print_function

import argparse
import base64
import calendar
import codecs
import errno
import getpass
import json
import logging
import os
import re
import select
import subprocess
import sys
import time
try:
    # python 3
    from urllib.parse import urlparse
    from urllib.parse import quote as urlquote
    from urllib.parse import urlencode
    from urllib.error import HTTPError, URLError
    from urllib.request import urlopen
    from urllib.request import Request
except ImportError:
    # python 2
    from urlparse import urlparse
    from urllib import quote as urlquote
    from urllib import urlencode
    from urllib2 import HTTPError, URLError
    from urllib2 import urlopen
    from urllib2 import Request

from github_backup import __version__

FNULL = open(os.devnull, 'w')


def log_error(message):
    if type(message) == str:
        message = [message]

    for msg in message:
        sys.stderr.write("{0}\n".format(msg))

    sys.exit(1)


def log_info(message):
    if type(message) == str:
        message = [message]

    for msg in message:
        sys.stdout.write("{0}\n".format(msg))


def logging_subprocess(popenargs,
                       logger,
                       stdout_log_level=logging.DEBUG,
                       stderr_log_level=logging.ERROR,
                       **kwargs):
    """
    Variant of subprocess.call that accepts a logger instead of stdout/stderr,
    and logs stdout messages via logger.debug and stderr messages via
    logger.error.
    """
    child = subprocess.Popen(popenargs, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, **kwargs)

    log_level = {child.stdout: stdout_log_level,
                 child.stderr: stderr_log_level}

    def check_io():
        ready_to_read = select.select([child.stdout, child.stderr],
                                      [],
                                      [],
                                      1000)[0]
        for io in ready_to_read:
            line = io.readline()
            if not logger:
                continue
            if not (io == child.stderr and not line):
                logger.log(log_level[io], line[:-1])

    # keep checking stdout/stderr until the child exits
    while child.poll() is None:
        check_io()

    check_io()  # check again to catch anything after the process exits

    rc = child.wait()

    if rc != 0:
        print('{} returned {}:'.format(popenargs[0], rc), file=sys.stderr)
        print('\t', ' '.join(popenargs), file=sys.stderr)

    return rc


def mkdir_p(*args):
    for path in args:
        try:
            os.makedirs(path)
        except OSError as exc:  # Python >2.5
            if exc.errno == errno.EEXIST and os.path.isdir(path):
                pass
            else:
                raise


def mask_password(url, secret='*****'):
    parsed = urlparse(url)

    if not parsed.password:
        return url
    elif parsed.password == 'x-oauth-basic':
        return url.replace(parsed.username, secret)

    return url.replace(parsed.password, secret)


def parse_args():
    parser = argparse.ArgumentParser(description='Backup a github account')
    parser.add_argument('user',
                        metavar='USER',
                        type=str,
                        help='github username')
    parser.add_argument('-u',
                        '--username',
                        dest='username',
                        help='username for basic auth')
    parser.add_argument('-p',
                        '--password',
                        dest='password',
                        help='password for basic auth. '
                             'If a username is given but not a password, the '
                             'password will be prompted for.')
    parser.add_argument('-t',
                        '--token',
                        dest='token',
                        help='personal access or OAuth token, or path to token (file://...)')  # noqa
    parser.add_argument('-o',
                        '--output-directory',
                        default='.',
                        dest='output_directory',
                        help='directory at which to backup the repositories')
    parser.add_argument('-i',
                        '--incremental',
                        action='store_true',
                        dest='incremental',
                        help='incremental backup')
    parser.add_argument('--starred',
                        action='store_true',
                        dest='include_starred',
                        help='include starred repositories in backup')
    parser.add_argument('--watched',
                        action='store_true',
                        dest='include_watched',
                        help='include watched repositories in backup')
    parser.add_argument('--all',
                        action='store_true',
                        dest='include_everything',
                        help='include everything in backup')
    parser.add_argument('--issues',
                        action='store_true',
                        dest='include_issues',
                        help='include issues in backup')
    parser.add_argument('--issue-comments',
                        action='store_true',
                        dest='include_issue_comments',
                        help='include issue comments in backup')
    parser.add_argument('--issue-events',
                        action='store_true',
                        dest='include_issue_events',
                        help='include issue events in backup')
    parser.add_argument('--pulls',
                        action='store_true',
                        dest='include_pulls',
                        help='include pull requests in backup')
    parser.add_argument('--pull-comments',
                        action='store_true',
                        dest='include_pull_comments',
                        help='include pull request review comments in backup')
    parser.add_argument('--pull-commits',
                        action='store_true',
                        dest='include_pull_commits',
                        help='include pull request commits in backup')
    parser.add_argument('--labels',
                        action='store_true',
                        dest='include_labels',
                        help='include labels in backup')
    parser.add_argument('--hooks',
                        action='store_true',
                        dest='include_hooks',
                        help='include hooks in backup (works only when authenticated)')  # noqa
    parser.add_argument('--milestones',
                        action='store_true',
                        dest='include_milestones',
                        help='include milestones in backup')
    parser.add_argument('--repositories',
                        action='store_true',
                        dest='include_repository',
                        help='include repository clone in backup')
    parser.add_argument('--bare',
                        action='store_true',
                        dest='bare_clone',
                        help='clone bare repositories')
    parser.add_argument('--wikis',
                        action='store_true',
                        dest='include_wiki',
                        help='include wiki clone in backup')
    parser.add_argument('--skip-existing',
                        action='store_true',
                        dest='skip_existing',
                        help='skip project if a backup directory exists')
    parser.add_argument('-L',
                        '--languages',
                        dest='languages',
                        help='only allow these languages',
                        nargs='*')
    parser.add_argument('-N',
                        '--name-regex',
                        dest='name_regex',
                        help='python regex to match names against')
    parser.add_argument('-H',
                        '--github-host',
                        dest='github_host',
                        help='GitHub Enterprise hostname')
    parser.add_argument('-O',
                        '--organization',
                        action='store_true',
                        dest='organization',
                        help='whether or not this is an organization user')
    parser.add_argument('-R',
                        '--repository',
                        dest='repository',
                        help='name of repository to limit backup to')
    parser.add_argument('-P', '--private',
                        action='store_true',
                        dest='private',
                        help='include private repositories')
    parser.add_argument('-F', '--fork',
                        action='store_true',
                        dest='fork',
                        help='include forked repositories')
    parser.add_argument('--prefer-ssh',
                        action='store_true',
                        help='Clone repositories using SSH instead of HTTPS')
    parser.add_argument('-v', '--version',
                        action='version',
                        version='%(prog)s ' + __version__)
    return parser.parse_args()


def get_auth(args, encode=True):
    auth = None

    if args.token:
        _path_specifier = 'file://'
        if args.token.startswith(_path_specifier):
            args.token = open(args.token[len(_path_specifier):],
                              'rt').readline().strip()
        auth = args.token + ':' + 'x-oauth-basic'
    elif args.username:
        if not args.password:
            args.password = getpass.getpass()
        if encode:
            password = args.password
        else:
            password = urlquote(args.password)
        auth = args.username + ':' + password
    elif args.password:
        log_error('You must specify a username for basic auth')

    if not auth:
        return None

    if not encode:
        return auth

    return base64.b64encode(auth.encode('ascii'))


def get_github_api_host(args):
    if args.github_host:
        host = args.github_host + '/api/v3'
    else:
        host = 'api.github.com'

    return host


def get_github_host(args):
    if args.github_host:
        host = args.github_host
    else:
        host = 'github.com'

    return host


def get_github_repo_url(args, repository):
    if args.prefer_ssh:
        return repository['ssh_url']

    auth = get_auth(args, False)
    if auth:
        repo_url = 'https://{0}@{1}/{2}/{3}.git'.format(
            auth,
            get_github_host(args),
            args.user,
            repository['name'])
    else:
        repo_url = repository['clone_url']

    return repo_url


def retrieve_data(args, template, query_args=None, single_request=False):
    auth = get_auth(args)
    query_args = get_query_args(query_args)
    per_page = 100
    page = 0
    data = []

    while True:
        page = page + 1
        request = _construct_request(per_page, page, query_args, template, auth)  # noqa
        r, errors = _get_response(request, auth, template)

        status_code = int(r.getcode())

        if status_code != 200:
            template = 'API request returned HTTP {0}: {1}'
            errors.append(template.format(status_code, r.reason))
            log_error(errors)

        response = json.loads(r.read().decode('utf-8'))
        if len(errors) == 0:
            if type(response) == list:
                data.extend(response)
                if len(response) < per_page:
                    break
            elif type(response) == dict and single_request:
                data.append(response)

        if len(errors) > 0:
            log_error(errors)

        if single_request:
            break

    return data


def get_query_args(query_args=None):
    if not query_args:
        query_args = {}
    return query_args


def _get_response(request, auth, template):
    retry_timeout = 3
    errors = []
    # We'll make requests in a loop so we can
    # delay and retry in the case of rate-limiting
    while True:
        should_continue = False
        try:
            r = urlopen(request)
        except HTTPError as exc:
            errors, should_continue = _request_http_error(exc, auth, errors)  # noqa
            r = exc
        except URLError:
            should_continue = _request_url_error(template, retry_timeout)
            if not should_continue:
                raise

        if should_continue:
            continue

        break
    return r, errors


def _construct_request(per_page, page, query_args, template, auth):
    querystring = urlencode(dict(list({
        'per_page': per_page,
        'page': page
    }.items()) + list(query_args.items())))

    request = Request(template + '?' + querystring)
    if auth is not None:
        request.add_header('Authorization', 'Basic '.encode('ascii') + auth)
    return request


def _request_http_error(exc, auth, errors):
    # HTTPError behaves like a Response so we can
    # check the status code and headers to see exactly
    # what failed.

    should_continue = False
    headers = exc.headers
    limit_remaining = int(headers.get('x-ratelimit-remaining', 0))

    if exc.code == 403 and limit_remaining < 1:
        # The X-RateLimit-Reset header includes a
        # timestamp telling us when the limit will reset
        # so we can calculate how long to wait rather
        # than inefficiently polling:
        gm_now = calendar.timegm(time.gmtime())
        reset = int(headers.get('x-ratelimit-reset', 0)) or gm_now
        # We'll never sleep for less than 10 seconds:
        delta = max(10, reset - gm_now)

        limit = headers.get('x-ratelimit-limit')
        print('Exceeded rate limit of {} requests; waiting {} seconds to reset'.format(limit, delta),  # noqa
              file=sys.stderr)

        if auth is None:
            print('Hint: Authenticate to raise your GitHub rate limit',
                  file=sys.stderr)

        time.sleep(delta)
        should_continue = True
    return errors, should_continue


def _request_url_error(template, retry_timeout):
    # Incase of a connection timing out, we can retry a few time
    # But we won't crash and not back-up the rest now
    log_info('{} timed out'.format(template))
    retry_timeout -= 1

    if retry_timeout >= 0:
        return True

    log_error('{} timed out to much, skipping!')
    return False


def retrieve_repositories(args):
    log_info('Retrieving repositories')
    single_request = False
    template = 'https://{0}/user/repos'.format(
        get_github_api_host(args))
    if args.organization:
        template = 'https://{0}/orgs/{1}/repos'.format(
            get_github_api_host(args),
            args.user)

    if args.repository:
        single_request = True
        template = 'https://{0}/repos/{1}/{2}'.format(
            get_github_api_host(args),
            args.user,
            args.repository)

    return retrieve_data(args, template, single_request=single_request)


def filter_repositories(args, unfiltered_repositories):
    log_info('Filtering repositories')

    repositories = []
    for r in unfiltered_repositories:
        if r['owner']['login'] == args.user:
            repositories.append(r)

    name_regex = None
    if args.name_regex:
        name_regex = re.compile(args.name_regex)

    languages = None
    if args.languages:
        languages = [x.lower() for x in args.languages]

    if not args.fork:
        repositories = [r for r in repositories if not r['fork']]
    if not args.private:
        repositories = [r for r in repositories if not r['private']]
    if languages:
        repositories = [r for r in repositories if r['language'] and r['language'].lower() in languages]  # noqa
    if name_regex:
        repositories = [r for r in repositories if name_regex.match(r['name'])]

    return repositories


def backup_repositories(args, output_directory, repositories):
    log_info('Backing up repositories')
    repos_template = 'https://{0}/repos'.format(get_github_api_host(args))

    if args.incremental:
        last_update = max(repository['updated_at'] for repository in repositories)  # noqa
        last_update_path = os.path.join(output_directory, 'last_update')
        if os.path.exists(last_update_path):
            args.since = open(last_update_path).read().strip()
        else:
            args.since = None
    else:
        args.since = None

    for repository in repositories:
        backup_cwd = os.path.join(output_directory, 'repositories')
        repo_cwd = os.path.join(backup_cwd, repository['name'])
        repo_dir = os.path.join(repo_cwd, 'repository')
        repo_url = get_github_repo_url(args, repository)

        if args.include_repository or args.include_everything:
            fetch_repository(repository['name'],
                             repo_url,
                             repo_dir,
                             skip_existing=args.skip_existing,
                             bare_clone=args.bare_clone)

        download_wiki = (args.include_wiki or args.include_everything)
        if repository['has_wiki'] and download_wiki:
            fetch_repository(repository['name'],
                             repo_url.replace('.git', '.wiki.git'),
                             os.path.join(repo_cwd, 'wiki'),
                             skip_existing=args.skip_existing,
                             bare_clone=args.bare_clone)

        if args.include_issues or args.include_everything:
            backup_issues(args, repo_cwd, repository, repos_template)

        if args.include_pulls or args.include_everything:
            backup_pulls(args, repo_cwd, repository, repos_template)

        if args.include_milestones or args.include_everything:
            backup_milestones(args, repo_cwd, repository, repos_template)

        if args.include_labels or args.include_everything:
            backup_labels(args, repo_cwd, repository, repos_template)

        if args.include_hooks or args.include_everything:
            backup_hooks(args, repo_cwd, repository, repos_template)

    if args.incremental:
        open(last_update_path, 'w').write(last_update)


def backup_issues(args, repo_cwd, repository, repos_template):
    has_issues_dir = os.path.isdir('{0}/issues/.git'.format(repo_cwd))
    if args.skip_existing and has_issues_dir:
        return

    log_info('Retrieving {0} issues'.format(repository['full_name']))
    issue_cwd = os.path.join(repo_cwd, 'issues')
    mkdir_p(repo_cwd, issue_cwd)

    issues = {}
    issues_skipped = 0
    issues_skipped_message = ''
    _issue_template = '{0}/{1}/issues'.format(repos_template,
                                              repository['full_name'])

    should_include_pulls = args.include_pulls or args.include_everything
    issue_states = ['open', 'closed']
    for issue_state in issue_states:
        query_args = {
            'filter': 'all',
            'state': issue_state
        }
        if args.since:
            query_args['since'] = args.since

        _issues = retrieve_data(args,
                                _issue_template,
                                query_args=query_args)
        for issue in _issues:
            # skip pull requests which are also returned as issues
            # if retrieving pull requests is requested as well
            if 'pull_request' in issue and should_include_pulls:
                issues_skipped += 1
                continue

            issues[issue['number']] = issue

    if issues_skipped:
        issues_skipped_message = ' (skipped {0} pull requests)'.format(
            issues_skipped)

    log_info('Saving {0} issues to disk{1}'.format(
        len(list(issues.keys())), issues_skipped_message))
    comments_template = _issue_template + '/{0}/comments'
    events_template = _issue_template + '/{0}/events'
    for number, issue in list(issues.items()):
        if args.include_issue_comments or args.include_everything:
            template = comments_template.format(number)
            issues[number]['comment_data'] = retrieve_data(args, template)
        if args.include_issue_events or args.include_everything:
            template = events_template.format(number)
            issues[number]['event_data'] = retrieve_data(args, template)

        issue_file = '{0}/{1}.json'.format(issue_cwd, number)
        with codecs.open(issue_file, 'w', encoding='utf-8') as f:
            json_dump(issue, f)


def backup_pulls(args, repo_cwd, repository, repos_template):
    has_pulls_dir = os.path.isdir('{0}/pulls/.git'.format(repo_cwd))
    if args.skip_existing and has_pulls_dir:
        return

    log_info('Retrieving {0} pull requests'.format(repository['full_name']))  # noqa
    pulls_cwd = os.path.join(repo_cwd, 'pulls')
    mkdir_p(repo_cwd, pulls_cwd)

    pulls = {}
    _pulls_template = '{0}/{1}/pulls'.format(repos_template,
                                             repository['full_name'])

    pull_states = ['open', 'closed']
    for pull_state in pull_states:
        query_args = {
            'filter': 'all',
            'state': pull_state,
            'sort': 'updated',
            'direction': 'desc',
        }

        # It'd be nice to be able to apply the args.since filter here...
        _pulls = retrieve_data(args,
                               _pulls_template,
                               query_args=query_args)
        for pull in _pulls:
            if not args.since or pull['updated_at'] >= args.since:
                pulls[pull['number']] = pull

    log_info('Saving {0} pull requests to disk'.format(
        len(list(pulls.keys()))))
    comments_template = _pulls_template + '/{0}/comments'
    commits_template = _pulls_template + '/{0}/commits'
    for number, pull in list(pulls.items()):
        if args.include_pull_comments or args.include_everything:
            template = comments_template.format(number)
            pulls[number]['comment_data'] = retrieve_data(args, template)
        if args.include_pull_commits or args.include_everything:
            template = commits_template.format(number)
            pulls[number]['commit_data'] = retrieve_data(args, template)

        pull_file = '{0}/{1}.json'.format(pulls_cwd, number)
        with codecs.open(pull_file, 'w', encoding='utf-8') as f:
            json_dump(pull, f)


def backup_milestones(args, repo_cwd, repository, repos_template):
    milestone_cwd = os.path.join(repo_cwd, 'milestones')
    if args.skip_existing and os.path.isdir(milestone_cwd):
        return

    log_info('Retrieving {0} milestones'.format(repository['full_name']))
    mkdir_p(repo_cwd, milestone_cwd)

    template = '{0}/{1}/milestones'.format(repos_template,
                                           repository['full_name'])

    query_args = {
        'state': 'all'
    }

    _milestones = retrieve_data(args, template, query_args=query_args)

    milestones = {}
    for milestone in _milestones:
        milestones[milestone['number']] = milestone

    log_info('Saving {0} milestones to disk'.format(
        len(list(milestones.keys()))))
    for number, milestone in list(milestones.items()):
        milestone_file = '{0}/{1}.json'.format(milestone_cwd, number)
        with codecs.open(milestone_file, 'w', encoding='utf-8') as f:
            json_dump(milestone, f)


def backup_labels(args, repo_cwd, repository, repos_template):
    label_cwd = os.path.join(repo_cwd, 'labels')
    output_file = '{0}/labels.json'.format(label_cwd)
    template = '{0}/{1}/labels'.format(repos_template,
                                       repository['full_name'])
    _backup_data(args,
                 'labels',
                 template,
                 output_file,
                 label_cwd)


def backup_hooks(args, repo_cwd, repository, repos_template):
    auth = get_auth(args)
    if not auth:
        log_info("Skipping hooks since no authentication provided")
        return
    hook_cwd = os.path.join(repo_cwd, 'hooks')
    output_file = '{0}/hooks.json'.format(hook_cwd)
    template = '{0}/{1}/hooks'.format(repos_template,
                                      repository['full_name'])
    try:
        _backup_data(args,
                     'hooks',
                     template,
                     output_file,
                     hook_cwd)
    except SystemExit:
        log_info("Unable to read hooks, skipping")


def fetch_repository(name,
                     remote_url,
                     local_dir,
                     skip_existing=False,
                     bare_clone=False):
    if bare_clone:
        if os.path.exists(local_dir):
            clone_exists = subprocess.check_output(['git',
                                                    'rev-parse',
                                                    '--is-bare-repository'],
                                                   cwd=local_dir) == "true\n"
        else:
            clone_exists = False
    else:
        clone_exists = os.path.exists(os.path.join(local_dir, '.git'))

    if clone_exists and skip_existing:
        return

    masked_remote_url = mask_password(remote_url)

    initialized = subprocess.call('git ls-remote ' + remote_url,
                                  stdout=FNULL,
                                  stderr=FNULL,
                                  shell=True)
    if initialized == 128:
        log_info("Skipping {0} ({1}) since it's not initialized".format(
            name, masked_remote_url))
        return

    if clone_exists:
        log_info('Updating {0} in {1}'.format(name, local_dir))
        git_command = ['git', 'remote', 'rm', 'origin']
        logging_subprocess(git_command, None, cwd=local_dir)
        git_command = ['git', 'remote', 'add', 'origin', remote_url]
        logging_subprocess(git_command, None, cwd=local_dir)
        git_command = ['git', 'fetch', '--all', '--tags', '--prune']
        logging_subprocess(git_command, None, cwd=local_dir)
    else:
        log_info('Cloning {0} repository from {1} to {2}'.format(
            name,
            masked_remote_url,
            local_dir))
        if bare_clone:
            git_command = ['git', 'clone', '--mirror', remote_url, local_dir]
        else:
            git_command = ['git', 'clone', remote_url, local_dir]
        logging_subprocess(git_command, None)


def backup_account(args, output_directory):
    account_cwd = os.path.join(output_directory, 'account')

    if args.include_starred or args.include_everything:
        output_file = '{0}/starred.json'.format(account_cwd)
        template = "https://{0}/users/{1}/starred"
        template = template.format(get_github_api_host(args), args.user)
        _backup_data(args,
                     'starred repositories',
                     template,
                     output_file,
                     account_cwd)

    if args.include_watched or args.include_everything:
        output_file = '{0}/watched.json'.format(account_cwd)
        template = "https://{0}/users/{1}/subscriptions"
        template = template.format(get_github_api_host(args), args.user)
        _backup_data(args,
                     'watched repositories',
                     template,
                     output_file,
                     account_cwd)


def _backup_data(args, name, template, output_file, output_directory):
    skip_existing = args.skip_existing
    if not skip_existing or not os.path.exists(output_file):
        log_info('Retrieving {0} {1}'.format(args.user, name))
        mkdir_p(output_directory)
        data = retrieve_data(args, template)

        log_info('Writing {0} {1} to disk'.format(len(data), name))
        with codecs.open(output_file, 'w', encoding='utf-8') as f:
            json_dump(data, f)


def json_dump(data, output_file):
    json.dump(data,
              output_file,
              ensure_ascii=False,
              sort_keys=True,
              indent=4,
              separators=(',', ': '))


def main():
    args = parse_args()

    output_directory = os.path.realpath(args.output_directory)
    if not os.path.isdir(output_directory):
        log_info('Create output directory {0}'.format(output_directory))
        mkdir_p(output_directory)

    log_info('Backing up user {0} to {1}'.format(args.user, output_directory))

    repositories = retrieve_repositories(args)
    repositories = filter_repositories(args, repositories)
    backup_repositories(args, output_directory, repositories)
    backup_account(args, output_directory)


if __name__ == '__main__':
    main()