From 123b1584daba4d4c13cce5b73a9fb4ccd4a229e8 Mon Sep 17 00:00:00 2001 From: Egor Tensin Date: Tue, 19 Feb 2019 21:41:28 +0300 Subject: refactor to make for proper code --- pull/__init__.py | 0 pull/definitions.py | 72 +++++++++++++++++ pull/pull.py | 223 ++++++++++++++++++++++++++++++++++++++++++++++++++++ pull/registry.py | 56 +++++++++++++ 4 files changed, 351 insertions(+) create mode 100644 pull/__init__.py create mode 100644 pull/definitions.py create mode 100644 pull/pull.py create mode 100644 pull/registry.py (limited to 'pull') diff --git a/pull/__init__.py b/pull/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pull/definitions.py b/pull/definitions.py new file mode 100644 index 0000000..43b55e4 --- /dev/null +++ b/pull/definitions.py @@ -0,0 +1,72 @@ +import os.path + + +DEFAULT_OWNER = 'Egor Tensin' +DEFAULT_GITHUB_USER = 'egor-tensin' +DEFAULT_BITBUCKET_USER = 'egor-tensin' + + +class Repo: + @staticmethod + def extract_repo_name(repo_id): + return os.path.basename(repo_id) + + def __init__(self, repo_id, clone_url, owner=None, desc=None, + homepage=None): + self.repo_id = repo_id + self.repo_name = self.extract_repo_name(repo_id) + self.clone_url = clone_url + if owner is None: + owner = DEFAULT_OWNER + self.owner = owner + if desc is None: + if homepage is not None: + desc = homepage + elif clone_url is not None: + desc = clone_url + else: + desc = self.repo_name + self.desc = desc + self.homepage = homepage + + +class GithubRepo(Repo): + def __init__(self, repo_id, clone_url=None, owner=None, desc=None, + homepage=None, github_user=DEFAULT_GITHUB_USER): + if clone_url is None: + clone_url = self.build_clone_url(github_user, repo_id) + if homepage is None: + homepage = self.build_homepage_url(github_user, repo_id) + super().__init__(repo_id, clone_url, owner=owner, desc=desc, + homepage=homepage) + + @staticmethod + def build_clone_url(user, repo_id): + name = Repo.extract_repo_name(repo_id) + return f'ssh://git@github.com/{user}/{name}.git' + + @staticmethod + def build_homepage_url(user, repo_id): + name = Repo.extract_repo_name(repo_id) + return f'https://github.com/{user}/{name}' + + +class BitbucketRepo(Repo): + def __init__(self, repo_id, clone_url=None, owner=None, desc=None, + homepage=None, bitbucket_user=DEFAULT_BITBUCKET_USER): + if clone_url is None: + clone_url = self.build_clone_url(bitbucket_user, repo_id) + if homepage is None: + homepage = self.build_homepage_url(bitbucket_user, repo_id) + super().__init__(repo_id, clone_url, owner=owner, desc=desc, + homepage=homepage) + + @staticmethod + def build_clone_url(user, repo_id): + name = Repo.extract_repo_name(repo_id) + return f'ssh://git@bitbucket.org/{user}/{name}.git' + + @staticmethod + def build_homepage_url(user, repo_id): + name = Repo.extract_repo_name(repo_id) + return f'https://bitbucket.org/{user}/{name.lower()}' diff --git a/pull/pull.py b/pull/pull.py new file mode 100644 index 0000000..5e123a9 --- /dev/null +++ b/pull/pull.py @@ -0,0 +1,223 @@ +from argparse import ArgumentParser +import contextlib +from enum import Enum +import logging +import os +import os.path +import shutil +import socket +import sys +import subprocess + +from pull.registry import MY_REPOS + + +env = os.environ.copy() +env['GIT_SSH_COMMAND'] = 'ssh -oStrictHostKeyChecking=no -oBatchMode=yes' + +DEFAULT_OUTPUT_DIR = 'output' + +DEFAULT_CGIT_CLONE_USER = 'egor' +DEFAULT_CGIT_CLONE_HOST = 'tensin-ext1.home' +DEFAULT_CGIT_CLONE_PORT = 8080 + + +def set_up_logging(): + logging.basicConfig( + level=logging.DEBUG, + datefmt='%Y-%m-%d %H:%M:%S', + format='%(asctime)s | %(levelname)s | %(message)s') + + +def parse_args(argv=None): + if argv is None: + argv = sys.argv[1:] + parser = ArgumentParser() + parser.add_argument('--output', default=DEFAULT_OUTPUT_DIR, + help='output directory path') + parser.add_argument('--cgit-user', default=DEFAULT_CGIT_CLONE_USER, + help='cgit clone username') + parser.add_argument('--cgit-host', default=DEFAULT_CGIT_CLONE_HOST, + help='cgit clone host') + parser.add_argument('--cgit-port', default=DEFAULT_CGIT_CLONE_PORT, + help='cgit clone port number', type=int) + return parser.parse_args(argv) + + +def check_output(*args, stdout=subprocess.PIPE): + result = subprocess.run(args, stdout=stdout, stderr=subprocess.STDOUT, + env=env, encoding='utf-8') + try: + result.check_returncode() + if stdout != subprocess.DEVNULL: + if result.stdout is None: + logging.debug('%s', args) + else: + logging.debug('%s\n%s', args, result.stdout) + return result.returncode == 0, result.stdout + except subprocess.CalledProcessError as e: + if stdout != subprocess.DEVNULL: + logging.error('%s\n%s', e, e.output) + return e.returncode == 0, e.output + + +def run(*args, discard_output=False): + if discard_output: + success, _ = check_output(*args, stdout=subprocess.DEVNULL) + else: + success, _ = check_output(*args) + return success + + +@contextlib.contextmanager +def chdir(new_cwd): + old_cwd = os.getcwd() + os.chdir(new_cwd) + try: + yield + finally: + os.chdir(old_cwd) + + +class CGit: + def __init__(self, user, host, port): + self.user = user + self.host = host + self.ip = socket.gethostbyname(self.host) + self.port = port + + def get_clone_url(self, repo): + return f'http://{self.user}@{self.ip}:{self.port}/git/{repo.repo_id}' + + +class CGitRC: + def __init__(self, cgit): + self.cgit = cgit + + def write(self, path, repo): + with open(path, 'w') as fd: + self._write_field(fd, 'clone-url', self._build_clone_url(repo)) + self._write_field(fd, 'owner', repo.owner) + self._write_field(fd, 'desc', repo.desc) + self._write_field(fd, 'homepage', repo.homepage) + + @staticmethod + def _write_field(fd, field, value): + if value is None: + return + fd.write(f'{field}={value}\n') + + def _build_clone_url(self, repo): + clone_urls = [] + if repo.clone_url is not None: + clone_urls.append(repo.clone_url) + clone_urls.append(self.cgit.get_clone_url(repo)) + clone_urls = ' '.join(clone_urls) + return clone_urls + + +class Output: + def __init__(self, output_dir, cgit): + self.output_dir = self._make_dir(output_dir) + self.cgitrc = CGitRC(cgit) + + @staticmethod + def _make_dir(rel_path): + abs_path = os.path.abspath(rel_path) + os.makedirs(abs_path, exist_ok=True) + return abs_path + + def get_repo_dir(self, repo): + return os.path.join(self.output_dir, repo.repo_id) + + def get_cgitrc_path(self, repo): + return os.path.join(self.get_repo_dir(repo), 'cgitrc') + + def pull(self, repo): + success = False + verdict = self.judge(repo) + if verdict is RepoVerdict.SHOULD_MIRROR: + success = self.mirror(repo) + elif verdict is RepoVerdict.SHOULD_UPDATE: + success = self.update(repo) + elif verdict is RepoVerdict.CANT_DECIDE: + success = False + else: + raise NotImplementedError(f'Unknown repository verdict: {verdict}') + if success: + self.cgitrc.write(self.get_cgitrc_path(repo), repo) + return success + + def judge(self, repo): + repo_dir = self.get_repo_dir(repo) + if not os.path.isdir(repo_dir): + return RepoVerdict.SHOULD_MIRROR + with chdir(repo_dir): + if not run('git', 'rev-parse', '--is-inside-work-tree', discard_output=True): + logging.warning('Not a repository, so going to mirror: %s', repo_dir) + return RepoVerdict.SHOULD_MIRROR + success, output = check_output('git', 'config', '--get', 'remote.origin.url') + if not success: + # Every repository managed by this script should have the + # 'origin' remote. If it doesn't, it's trash. + return RepoVerdict.SHOULD_MIRROR + if f'{repo.clone_url}\n' != output: + logging.warning("Existing repository '%s' URL doesn't match the specified clone" \ + " URL: %s", repo.repo_id, repo.clone_url) + return RepoVerdict.CANT_DECIDE + # Looks like a legit clone of the specified remote. + return RepoVerdict.SHOULD_UPDATE + + def mirror(self, repo): + logging.info("Mirroring repository '%s' from: %s", repo.repo_id, + repo.clone_url) + repo_dir = self.get_repo_dir(repo) + if os.path.isdir(repo_dir): + try: + shutil.rmtree(repo_dir) + except Exception as e: + logging.exception(e) + return False + return run('git', 'clone', '--mirror', repo.clone_url, repo_dir) + + def update(self, repo): + logging.info("Updating repository '%s'", repo.repo_id) + repo_dir = self.get_repo_dir(repo) + with chdir(repo_dir): + if not run('git', 'remote', 'update', '--prune'): + return False + if run('git', 'rev-parse', '--verify', '--quiet', 'origin/master', discard_output=True): + if not run('git', 'reset', '--soft', 'origin/master'): + return False + return True + + +class RepoVerdict(Enum): + SHOULD_MIRROR = 1 + SHOULD_UPDATE = 2 + CANT_DECIDE = 3 + + +def main(args=None): + set_up_logging() + try: + args = parse_args(args) + cgit = CGit(args.cgit_user, args.cgit_host, args.cgit_port) + output = Output(args.output, cgit) + success = True + for repo in MY_REPOS: + if not output.pull(repo): + success = False + if success: + logging.info('All repositories were updated successfully') + return 0 + else: + logging.warning("Some repositories couldn't be updated!") + return 1 + except Exception as e: + logging.exception(e) + raise + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/pull/registry.py b/pull/registry.py new file mode 100644 index 0000000..0e3e951 --- /dev/null +++ b/pull/registry.py @@ -0,0 +1,56 @@ +from pull.definitions import BitbucketRepo, GithubRepo, Repo + + +MY_REPOS = ( + GithubRepo('personal/aes-tools'), + GithubRepo('personal/blog'), + GithubRepo('personal/chess-games'), + GithubRepo('personal/cmake-common'), + GithubRepo('personal/config-links'), + GithubRepo('personal/cv'), + GithubRepo('personal/egor-tensin.github.io'), + GithubRepo('personal/filters'), + GithubRepo('personal/linux-home'), + GithubRepo('personal/linux-status'), + GithubRepo('personal/notes'), + GithubRepo('personal/pdb-repo'), + GithubRepo('personal/privilege-check'), + GithubRepo('personal/simple-interpreter'), + GithubRepo('personal/sorting-algorithms'), + GithubRepo('personal/vk-scripts'), + GithubRepo('personal/windows-env'), + GithubRepo('personal/windows-home'), + GithubRepo('personal/windows-tmp'), + GithubRepo('personal/windows7-drivers'), + GithubRepo('personal/writable-dirs'), + + BitbucketRepo('etc/etc-tensin-laptop1'), + BitbucketRepo('etc/etc-tensin-laptop2'), + BitbucketRepo('etc/etc-tensin-pc1'), + BitbucketRepo('etc/etc-tensin-raspi1'), + BitbucketRepo('etc/etc-tensin-raspi2'), + BitbucketRepo('fr24/fr24-cover-letter'), + BitbucketRepo('fr24/fr24-home'), + BitbucketRepo('fr24/fr24-tmp'), + BitbucketRepo('netwrix/etc-wiki'), + BitbucketRepo('netwrix/netwrix-copyright'), + BitbucketRepo('netwrix/netwrix-lab'), + BitbucketRepo('netwrix/netwrix-logs'), + #BitbucketRepo('netwrix/netwrix-webapi'), + BitbucketRepo('netwrix/netwrix-xml'), + BitbucketRepo('netwrix/netwrix.sh'), + BitbucketRepo('netwrix/wiki-backup'), + BitbucketRepo('shadow'), + BitbucketRepo('staging/361_Tensin_E_D_report'), + BitbucketRepo('staging/361_Tensin_E_D_slides'), + BitbucketRepo('staging/461_Tensin_E_D_report'), + BitbucketRepo('staging/461_Tensin_E_D_slides'), + BitbucketRepo('staging/cgit-repos'), + BitbucketRepo('staging/deposit-calculator'), + BitbucketRepo('staging/raspi-temp-client'), + BitbucketRepo('staging/raspi-temp-server'), + BitbucketRepo('staging/x64-decoder'), + + Repo('fr24/key_mgmt', 'ssh://egor@tensin-raspi2/~/tmp/key_mgmt.git'), + Repo('fr24/openfortivpn', 'ssh://egor@tensin-raspi2/~/tmp/openfortivpn.git'), +) -- cgit v1.2.3