aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authorEgor Tensin <Egor.Tensin@gmail.com>2021-08-01 16:10:47 +0300
committerEgor Tensin <Egor.Tensin@gmail.com>2021-08-01 16:24:05 +0300
commit4014dc5e8fb0194a88c16f6ee2db7a6682ffab52 (patch)
tree4289952fd652eb63b75ea1fdbf4180809d694e73
parentfix the lately broken --repo flag (diff)
downloadcgitize-4014dc5e8fb0194a88c16f6ee2db7a6682ffab52.tar.gz
cgitize-4014dc5e8fb0194a88c16f6ee2db7a6682ffab52.zip
support mirroring all of user's repositories
-rw-r--r--.github/workflows/ci.yml6
-rw-r--r--cgitize/bitbucket.py21
-rw-r--r--cgitize/cgit.py2
-rw-r--r--cgitize/config.py44
-rw-r--r--cgitize/github.py14
-rw-r--r--cgitize/repo.py23
-rw-r--r--examples/cgitize.toml12
-rwxr-xr-xtest/integration/example/test.sh18
8 files changed, 120 insertions, 20 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c4d1aed..346a1aa 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -5,6 +5,12 @@ on:
pull_request:
workflow_dispatch:
+env:
+ # API request rate limit is exceeded otherwise:
+ CGITIZE_GITHUB_ACCESS_TOKEN: '${{ secrets.CGITIZE_GITHUB_ACCESS_TOKEN }}'
+ CGITIZE_BITBUCKET_USERNAME: '${{ secrets.CGITIZE_BITBUCKET_USERNAME }}'
+ CGITIZE_BITBUCKET_APP_PASSWORD: '${{ secrets.CGITIZE_BITBUCKET_APP_PASSWORD }}'
+
jobs:
test_local:
runs-on: ubuntu-latest
diff --git a/cgitize/bitbucket.py b/cgitize/bitbucket.py
index 16f6d41..eaa8bb5 100644
--- a/cgitize/bitbucket.py
+++ b/cgitize/bitbucket.py
@@ -14,11 +14,22 @@ class Bitbucket:
self._impl = Cloud(username=username, password=password, cloud=True)
def get_repo(self, repo):
- if 'id' not in repo:
- raise ValueError('every Bitbucket repository must have an ID')
- repo_id = repo['id']
try:
- return self._impl.repositories.get(repo_id)
+ return self._impl.repositories.get(repo.id)
except HTTPError:
- logging.error("Couldn't fetch repository: %s", repo_id)
+ logging.error("Couldn't fetch repository: %s", repo.id)
+ raise
+
+ def get_user_repos(self, user):
+ try:
+ page = 1
+ while True:
+ params = {'page': page}
+ response = self._impl.repositories.get(user.name, params=params)
+ yield from iter(response['values'])
+ if 'next' not in response:
+ break
+ page += 1
+ except HTTPError:
+ logging.error("Couldn't fetch user repositories: %s", user.name)
raise
diff --git a/cgitize/cgit.py b/cgitize/cgit.py
index 418f0c6..7f07876 100644
--- a/cgitize/cgit.py
+++ b/cgitize/cgit.py
@@ -96,7 +96,7 @@ class CGitRepositories:
return abs_path
def get_repo_dir(self, repo):
- return os.path.join(self.dir, repo.name)
+ return os.path.join(self.dir, repo.dir)
def update(self, repo):
success = self._mirror_or_update(repo)
diff --git a/cgitize/config.py b/cgitize/config.py
index 8302f29..35ba05f 100644
--- a/cgitize/config.py
+++ b/cgitize/config.py
@@ -57,6 +57,7 @@ class MainSection(Section):
class GitHubSection(Section):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
+ self.users = UsersSection(self.impl.get('users', {}))
self.repositories = RepositoriesSection(self.impl.get('repositories', {}))
@property
@@ -71,6 +72,7 @@ class GitHubSection(Section):
class BitbucketSection(Section):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
+ self.users = UsersSection(self.impl.get('users', {}))
self.repositories = RepositoriesSection(self.impl.get('repositories', {}))
@property
@@ -89,12 +91,46 @@ class BitbucketSection(Section):
return None
return f'{username}:{password}'
+ def enum_repositories(self):
+ return map(HostedRepo, self.repositories.enum_repositories())
+
+
+class UsersSection(Section):
+ def enum_users(self):
+ return self.impl.values()
+
class RepositoriesSection(Section):
def enum_repositories(self):
return self.impl.values()
+class User:
+ def __init__(self, impl):
+ if 'name' not in impl:
+ raise ValueError("every user must have 'name'")
+ self._impl = impl
+
+ @property
+ def name(self):
+ return self._impl['name']
+
+ @property
+ def dir(self):
+ return self._impl.get('dir')
+
+
+class HostedRepo:
+ def __init__(self, impl):
+ if 'id' not in impl:
+ raise ValueError("every hosted repository must have 'id'")
+ self._impl = impl
+
+ @property
+ def id(self):
+ return self._impl['id']
+
+
class Config:
DEFAULT_PATH = '/etc/cgitize/cgitize.toml'
@@ -118,12 +154,20 @@ class Config:
def _parse_github_repositories(self):
github = GitHub(self.github.access_token)
for r in self.github.repositories.enum_repositories():
+ r = HostedRepo(r)
yield Repo.from_github(github.get_repo(r), self)
+ for u in self.github.users.enum_users():
+ u = User(u)
+ yield from (Repo.from_github(r, self, u.dir) for r in github.get_user_repos(u))
def _parse_bitbucket_repositories(self):
bitbucket = Bitbucket(self.bitbucket.username, self.bitbucket.app_password)
for r in self.bitbucket.repositories.enum_repositories():
+ r = HostedRepo(r)
yield Repo.from_bitbucket(bitbucket.get_repo(r), self)
+ for u in self.bitbucket.users.enum_users():
+ u = User(u)
+ yield from (Repo.from_bitbucket(r, self, u.dir) for r in bitbucket.get_user_repos(u))
def parse_repositories(self):
yield from self._parse_explicit_repositories()
diff --git a/cgitize/github.py b/cgitize/github.py
index 2c88426..a5754af 100644
--- a/cgitize/github.py
+++ b/cgitize/github.py
@@ -13,11 +13,15 @@ class GitHub:
self._impl = Github(access_token)
def get_repo(self, repo):
- if 'id' not in repo:
- raise ValueError('every GitHub repository must have an ID')
- repo_id = repo['id']
try:
- return self._impl.get_repo(repo_id)
+ return self._impl.get_repo(repo.id)
except GithubException:
- logging.error("Couldn't fetch repository: %s", repo_id)
+ logging.error("Couldn't fetch repository: %s", repo.id)
+ raise
+
+ def get_user_repos(self, user):
+ try:
+ return self._impl.get_user(user.name).get_repos()
+ except GithubException:
+ logging.error("Couldn't fetch user repositories: %s", user.name)
raise
diff --git a/cgitize/repo.py b/cgitize/repo.py
index 99f39cf..cce3209 100644
--- a/cgitize/repo.py
+++ b/cgitize/repo.py
@@ -3,6 +3,8 @@
# For details, see https://github.com/egor-tensin/cgitize.
# Distributed under the MIT License.
+import os.path
+
from cgitize.utils import url_remove_auth, url_replace_auth
@@ -10,18 +12,18 @@ class Repo:
@staticmethod
def from_config(src, config):
if 'name' not in src:
- raise ValueError('every repository must have a name')
+ raise ValueError("every repository must have 'name'")
name = src['name']
desc = src.get('desc')
homepage = src.get('homepage')
owner = src.get('owner', config.main.default_owner)
if 'clone_url' not in src:
- raise ValueError('every repository must have a clone URL')
+ raise ValueError("every repository must have 'clone_url'")
clone_url = src['clone_url']
return Repo(name, clone_url, owner=owner, desc=desc, homepage=homepage)
@staticmethod
- def from_github(src, config):
+ def from_github(src, config, subdir=None):
name = src.name
desc = src.description
homepage = src.html_url
@@ -33,10 +35,10 @@ class Repo:
url_auth = None if config.main.clone_via_ssh else config.github.url_auth
return Repo(name, clone_url, owner=owner, desc=desc, homepage=homepage,
- url_auth=url_auth)
+ url_auth=url_auth, subdir=subdir)
@staticmethod
- def from_bitbucket(src, config):
+ def from_bitbucket(src, config, subdir=None):
name = src['name']
desc = src['description']
homepage = src['links']['html']['href']
@@ -56,16 +58,17 @@ class Repo:
url_auth = None if config.main.clone_via_ssh else config.bitbucket.url_auth
return Repo(name, clone_url, owner=owner, desc=desc, homepage=homepage,
- url_auth=url_auth)
+ url_auth=url_auth, subdir=subdir)
def __init__(self, name, clone_url, owner=None, desc=None, homepage=None,
- url_auth=None):
+ url_auth=None, subdir=None):
self._name = name
self._desc = desc
self._homepage = homepage
self._owner = owner
self._clone_url = clone_url
self._url_auth = url_auth
+ self._dir = subdir
@property
def name(self):
@@ -102,3 +105,9 @@ class Repo:
if not self.url_auth:
return self.clone_url
return url_replace_auth(self.clone_url, self.url_auth)
+
+ @property
+ def dir(self):
+ if self._dir is None:
+ return self.name
+ return os.path.join(self._dir, self.name)
diff --git a/examples/cgitize.toml b/examples/cgitize.toml
index b029328..19b7056 100644
--- a/examples/cgitize.toml
+++ b/examples/cgitize.toml
@@ -20,6 +20,12 @@ owner = "Your Name"
# CGITIZE_GITHUB_ACCESS_TOKEN environment variable.
#access_token = "XXX"
+[github.users.me]
+name = "egor-tensin"
+# To avoid directory name clashes with other repositories, you can specify a
+# subdirectory to put that user's repositories to.
+dir = "github-dir"
+
# Some random repositories hosted on GitHub:
[github.repositories.lens]
id = "ekmett/lens"
@@ -36,6 +42,12 @@ id = "bobfang1992/pytomlpp"
# environment variable).
#username = "your-username"
+[bitbucket.users.me]
+name = "egor-tensin"
+# To avoid directory name clashes with other repositories, you can specify a
+# subdirectory to put that user's repositories to.
+dir = "bitbucket-dir"
+
# Some random repositories hosted on Bitbucket:
[bitbucket.repositories.cef]
id = "chromiumembedded/cef"
diff --git a/test/integration/example/test.sh b/test/integration/example/test.sh
index 9345360..b8135f5 100755
--- a/test/integration/example/test.sh
+++ b/test/integration/example/test.sh
@@ -120,9 +120,16 @@ test_ssh() {
setup_ssh
cgitize
- verify_repos lens cef wintun
+ verify_repos \
+ lens \
+ cef \
+ wintun \
+ github-dir/cgitize-test-repository \
+ bitbucket-dir/cgitize-test-repository
verify_origin lens 'git@github.com:ekmett/lens.git'
verify_origin cef 'git@bitbucket.org:chromiumembedded/cef.git'
+ verify_origin github-dir/cgitize-test-repository 'git@github.com:egor-tensin/cgitize-test-repository.git'
+ verify_origin bitbucket-dir/cgitize-test-repository 'git@bitbucket.org:egor-tensin/cgitize-test-repository.git'
cleanup
}
@@ -134,9 +141,16 @@ test_https() {
setup_https
cgitize
- verify_repos lens cef wintun
+ verify_repos \
+ lens \
+ cef \
+ wintun \
+ github-dir/cgitize-test-repository \
+ bitbucket-dir/cgitize-test-repository
verify_origin lens 'https://github.com/ekmett/lens.git'
verify_origin cef 'https://bitbucket.org/chromiumembedded/cef.git'
+ verify_origin github-dir/cgitize-test-repository 'https://github.com/egor-tensin/cgitize-test-repository.git'
+ verify_origin bitbucket-dir/cgitize-test-repository 'https://bitbucket.org/egor-tensin/cgitize-test-repository.git'
cleanup
}