diff options
author | Egor Tensin <Egor.Tensin@gmail.com> | 2021-08-01 16:10:47 +0300 |
---|---|---|
committer | Egor Tensin <Egor.Tensin@gmail.com> | 2021-08-01 16:24:05 +0300 |
commit | 4014dc5e8fb0194a88c16f6ee2db7a6682ffab52 (patch) | |
tree | 4289952fd652eb63b75ea1fdbf4180809d694e73 | |
parent | fix the lately broken --repo flag (diff) | |
download | cgitize-4014dc5e8fb0194a88c16f6ee2db7a6682ffab52.tar.gz cgitize-4014dc5e8fb0194a88c16f6ee2db7a6682ffab52.zip |
support mirroring all of user's repositories
-rw-r--r-- | .github/workflows/ci.yml | 6 | ||||
-rw-r--r-- | cgitize/bitbucket.py | 21 | ||||
-rw-r--r-- | cgitize/cgit.py | 2 | ||||
-rw-r--r-- | cgitize/config.py | 44 | ||||
-rw-r--r-- | cgitize/github.py | 14 | ||||
-rw-r--r-- | cgitize/repo.py | 23 | ||||
-rw-r--r-- | examples/cgitize.toml | 12 | ||||
-rwxr-xr-x | test/integration/example/test.sh | 18 |
8 files changed, 120 insertions, 20 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4d1aed..346a1aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,12 @@ on: pull_request: workflow_dispatch: +env: + # API request rate limit is exceeded otherwise: + CGITIZE_GITHUB_ACCESS_TOKEN: '${{ secrets.CGITIZE_GITHUB_ACCESS_TOKEN }}' + CGITIZE_BITBUCKET_USERNAME: '${{ secrets.CGITIZE_BITBUCKET_USERNAME }}' + CGITIZE_BITBUCKET_APP_PASSWORD: '${{ secrets.CGITIZE_BITBUCKET_APP_PASSWORD }}' + jobs: test_local: runs-on: ubuntu-latest diff --git a/cgitize/bitbucket.py b/cgitize/bitbucket.py index 16f6d41..eaa8bb5 100644 --- a/cgitize/bitbucket.py +++ b/cgitize/bitbucket.py @@ -14,11 +14,22 @@ class Bitbucket: self._impl = Cloud(username=username, password=password, cloud=True) def get_repo(self, repo): - if 'id' not in repo: - raise ValueError('every Bitbucket repository must have an ID') - repo_id = repo['id'] try: - return self._impl.repositories.get(repo_id) + return self._impl.repositories.get(repo.id) except HTTPError: - logging.error("Couldn't fetch repository: %s", repo_id) + logging.error("Couldn't fetch repository: %s", repo.id) + raise + + def get_user_repos(self, user): + try: + page = 1 + while True: + params = {'page': page} + response = self._impl.repositories.get(user.name, params=params) + yield from iter(response['values']) + if 'next' not in response: + break + page += 1 + except HTTPError: + logging.error("Couldn't fetch user repositories: %s", user.name) raise diff --git a/cgitize/cgit.py b/cgitize/cgit.py index 418f0c6..7f07876 100644 --- a/cgitize/cgit.py +++ b/cgitize/cgit.py @@ -96,7 +96,7 @@ class CGitRepositories: return abs_path def get_repo_dir(self, repo): - return os.path.join(self.dir, repo.name) + return os.path.join(self.dir, repo.dir) def update(self, repo): success = self._mirror_or_update(repo) diff --git a/cgitize/config.py b/cgitize/config.py index 8302f29..35ba05f 100644 --- a/cgitize/config.py +++ b/cgitize/config.py @@ -57,6 +57,7 @@ class MainSection(Section): class GitHubSection(Section): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.users = UsersSection(self.impl.get('users', {})) self.repositories = RepositoriesSection(self.impl.get('repositories', {})) @property @@ -71,6 +72,7 @@ class GitHubSection(Section): class BitbucketSection(Section): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.users = UsersSection(self.impl.get('users', {})) self.repositories = RepositoriesSection(self.impl.get('repositories', {})) @property @@ -89,12 +91,46 @@ class BitbucketSection(Section): return None return f'{username}:{password}' + def enum_repositories(self): + return map(HostedRepo, self.repositories.enum_repositories()) + + +class UsersSection(Section): + def enum_users(self): + return self.impl.values() + class RepositoriesSection(Section): def enum_repositories(self): return self.impl.values() +class User: + def __init__(self, impl): + if 'name' not in impl: + raise ValueError("every user must have 'name'") + self._impl = impl + + @property + def name(self): + return self._impl['name'] + + @property + def dir(self): + return self._impl.get('dir') + + +class HostedRepo: + def __init__(self, impl): + if 'id' not in impl: + raise ValueError("every hosted repository must have 'id'") + self._impl = impl + + @property + def id(self): + return self._impl['id'] + + class Config: DEFAULT_PATH = '/etc/cgitize/cgitize.toml' @@ -118,12 +154,20 @@ class Config: def _parse_github_repositories(self): github = GitHub(self.github.access_token) for r in self.github.repositories.enum_repositories(): + r = HostedRepo(r) yield Repo.from_github(github.get_repo(r), self) + for u in self.github.users.enum_users(): + u = User(u) + yield from (Repo.from_github(r, self, u.dir) for r in github.get_user_repos(u)) def _parse_bitbucket_repositories(self): bitbucket = Bitbucket(self.bitbucket.username, self.bitbucket.app_password) for r in self.bitbucket.repositories.enum_repositories(): + r = HostedRepo(r) yield Repo.from_bitbucket(bitbucket.get_repo(r), self) + for u in self.bitbucket.users.enum_users(): + u = User(u) + yield from (Repo.from_bitbucket(r, self, u.dir) for r in bitbucket.get_user_repos(u)) def parse_repositories(self): yield from self._parse_explicit_repositories() diff --git a/cgitize/github.py b/cgitize/github.py index 2c88426..a5754af 100644 --- a/cgitize/github.py +++ b/cgitize/github.py @@ -13,11 +13,15 @@ class GitHub: self._impl = Github(access_token) def get_repo(self, repo): - if 'id' not in repo: - raise ValueError('every GitHub repository must have an ID') - repo_id = repo['id'] try: - return self._impl.get_repo(repo_id) + return self._impl.get_repo(repo.id) except GithubException: - logging.error("Couldn't fetch repository: %s", repo_id) + logging.error("Couldn't fetch repository: %s", repo.id) + raise + + def get_user_repos(self, user): + try: + return self._impl.get_user(user.name).get_repos() + except GithubException: + logging.error("Couldn't fetch user repositories: %s", user.name) raise diff --git a/cgitize/repo.py b/cgitize/repo.py index 99f39cf..cce3209 100644 --- a/cgitize/repo.py +++ b/cgitize/repo.py @@ -3,6 +3,8 @@ # For details, see https://github.com/egor-tensin/cgitize. # Distributed under the MIT License. +import os.path + from cgitize.utils import url_remove_auth, url_replace_auth @@ -10,18 +12,18 @@ class Repo: @staticmethod def from_config(src, config): if 'name' not in src: - raise ValueError('every repository must have a name') + raise ValueError("every repository must have 'name'") name = src['name'] desc = src.get('desc') homepage = src.get('homepage') owner = src.get('owner', config.main.default_owner) if 'clone_url' not in src: - raise ValueError('every repository must have a clone URL') + raise ValueError("every repository must have 'clone_url'") clone_url = src['clone_url'] return Repo(name, clone_url, owner=owner, desc=desc, homepage=homepage) @staticmethod - def from_github(src, config): + def from_github(src, config, subdir=None): name = src.name desc = src.description homepage = src.html_url @@ -33,10 +35,10 @@ class Repo: url_auth = None if config.main.clone_via_ssh else config.github.url_auth return Repo(name, clone_url, owner=owner, desc=desc, homepage=homepage, - url_auth=url_auth) + url_auth=url_auth, subdir=subdir) @staticmethod - def from_bitbucket(src, config): + def from_bitbucket(src, config, subdir=None): name = src['name'] desc = src['description'] homepage = src['links']['html']['href'] @@ -56,16 +58,17 @@ class Repo: url_auth = None if config.main.clone_via_ssh else config.bitbucket.url_auth return Repo(name, clone_url, owner=owner, desc=desc, homepage=homepage, - url_auth=url_auth) + url_auth=url_auth, subdir=subdir) def __init__(self, name, clone_url, owner=None, desc=None, homepage=None, - url_auth=None): + url_auth=None, subdir=None): self._name = name self._desc = desc self._homepage = homepage self._owner = owner self._clone_url = clone_url self._url_auth = url_auth + self._dir = subdir @property def name(self): @@ -102,3 +105,9 @@ class Repo: if not self.url_auth: return self.clone_url return url_replace_auth(self.clone_url, self.url_auth) + + @property + def dir(self): + if self._dir is None: + return self.name + return os.path.join(self._dir, self.name) diff --git a/examples/cgitize.toml b/examples/cgitize.toml index b029328..19b7056 100644 --- a/examples/cgitize.toml +++ b/examples/cgitize.toml @@ -20,6 +20,12 @@ owner = "Your Name" # CGITIZE_GITHUB_ACCESS_TOKEN environment variable. #access_token = "XXX" +[github.users.me] +name = "egor-tensin" +# To avoid directory name clashes with other repositories, you can specify a +# subdirectory to put that user's repositories to. +dir = "github-dir" + # Some random repositories hosted on GitHub: [github.repositories.lens] id = "ekmett/lens" @@ -36,6 +42,12 @@ id = "bobfang1992/pytomlpp" # environment variable). #username = "your-username" +[bitbucket.users.me] +name = "egor-tensin" +# To avoid directory name clashes with other repositories, you can specify a +# subdirectory to put that user's repositories to. +dir = "bitbucket-dir" + # Some random repositories hosted on Bitbucket: [bitbucket.repositories.cef] id = "chromiumembedded/cef" diff --git a/test/integration/example/test.sh b/test/integration/example/test.sh index 9345360..b8135f5 100755 --- a/test/integration/example/test.sh +++ b/test/integration/example/test.sh @@ -120,9 +120,16 @@ test_ssh() { setup_ssh cgitize - verify_repos lens cef wintun + verify_repos \ + lens \ + cef \ + wintun \ + github-dir/cgitize-test-repository \ + bitbucket-dir/cgitize-test-repository verify_origin lens 'git@github.com:ekmett/lens.git' verify_origin cef 'git@bitbucket.org:chromiumembedded/cef.git' + verify_origin github-dir/cgitize-test-repository 'git@github.com:egor-tensin/cgitize-test-repository.git' + verify_origin bitbucket-dir/cgitize-test-repository 'git@bitbucket.org:egor-tensin/cgitize-test-repository.git' cleanup } @@ -134,9 +141,16 @@ test_https() { setup_https cgitize - verify_repos lens cef wintun + verify_repos \ + lens \ + cef \ + wintun \ + github-dir/cgitize-test-repository \ + bitbucket-dir/cgitize-test-repository verify_origin lens 'https://github.com/ekmett/lens.git' verify_origin cef 'https://bitbucket.org/chromiumembedded/cef.git' + verify_origin github-dir/cgitize-test-repository 'https://github.com/egor-tensin/cgitize-test-repository.git' + verify_origin bitbucket-dir/cgitize-test-repository 'https://bitbucket.org/egor-tensin/cgitize-test-repository.git' cleanup } |