support mirroring all of user's repositories

8 files changed, 120 insertions, 20 deletions

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c4d1aed..346a1aa 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -5,6 +5,12 @@ on:
   pull_request:
   workflow_dispatch:
 
+env:
+  # API request rate limit is exceeded otherwise:
+  CGITIZE_GITHUB_ACCESS_TOKEN: '${{ secrets.CGITIZE_GITHUB_ACCESS_TOKEN }}'
+  CGITIZE_BITBUCKET_USERNAME: '${{ secrets.CGITIZE_BITBUCKET_USERNAME }}'
+  CGITIZE_BITBUCKET_APP_PASSWORD: '${{ secrets.CGITIZE_BITBUCKET_APP_PASSWORD }}'
+
 jobs:
   test_local:
     runs-on: ubuntu-latest
diff --git a/cgitize/bitbucket.py b/cgitize/bitbucket.py
index 16f6d41..eaa8bb5 100644
--- a/cgitize/bitbucket.py
+++ b/cgitize/bitbucket.py
@@ -14,11 +14,22 @@ class Bitbucket:
         self._impl = Cloud(username=username, password=password, cloud=True)
 
     def get_repo(self, repo):
-        if 'id' not in repo:
-            raise ValueError('every Bitbucket repository must have an ID')
-        repo_id = repo['id']
         try:
-            return self._impl.repositories.get(repo_id)
+            return self._impl.repositories.get(repo.id)
         except HTTPError:
-            logging.error("Couldn't fetch repository: %s", repo_id)
+            logging.error("Couldn't fetch repository: %s", repo.id)
+            raise
+
+    def get_user_repos(self, user):
+        try:
+            page = 1
+            while True:
+                params = {'page': page}
+                response = self._impl.repositories.get(user.name, params=params)
+                yield from iter(response['values'])
+                if 'next' not in response:
+                    break
+                page += 1
+        except HTTPError:
+            logging.error("Couldn't fetch user repositories: %s", user.name)
             raise
diff --git a/cgitize/cgit.py b/cgitize/cgit.py
index 418f0c6..7f07876 100644
--- a/cgitize/cgit.py
+++ b/cgitize/cgit.py
@@ -96,7 +96,7 @@ class CGitRepositories:
         return abs_path
 
     def get_repo_dir(self, repo):
-        return os.path.join(self.dir, repo.name)
+        return os.path.join(self.dir, repo.dir)
 
     def update(self, repo):
         success = self._mirror_or_update(repo)
diff --git a/cgitize/config.py b/cgitize/config.py
index 8302f29..35ba05f 100644
--- a/cgitize/config.py
+++ b/cgitize/config.py
@@ -57,6 +57,7 @@ class MainSection(Section):
 class GitHubSection(Section):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self.users = UsersSection(self.impl.get('users', {}))
         self.repositories = RepositoriesSection(self.impl.get('repositories', {}))
 
     @property
@@ -71,6 +72,7 @@ class GitHubSection(Section):
 class BitbucketSection(Section):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self.users = UsersSection(self.impl.get('users', {}))
         self.repositories = RepositoriesSection(self.impl.get('repositories', {}))
 
     @property
@@ -89,12 +91,46 @@ class BitbucketSection(Section):
             return None
         return f'{username}:{password}'
 
+    def enum_repositories(self):
+        return map(HostedRepo, self.repositories.enum_repositories())
+
+
+class UsersSection(Section):
+    def enum_users(self):
+        return self.impl.values()
+
 
 class RepositoriesSection(Section):
     def enum_repositories(self):
         return self.impl.values()
 
 
+class User:
+    def __init__(self, impl):
+        if 'name' not in impl:
+            raise ValueError("every user must have 'name'")
+        self._impl = impl
+
+    @property
+    def name(self):
+        return self._impl['name']
+
+    @property
+    def dir(self):
+        return self._impl.get('dir')
+
+
+class HostedRepo:
+    def __init__(self, impl):
+        if 'id' not in impl:
+            raise ValueError("every hosted repository must have 'id'")
+        self._impl = impl
+
+    @property
+    def id(self):
+        return self._impl['id']
+
+
 class Config:
     DEFAULT_PATH = '/etc/cgitize/cgitize.toml'
 
@@ -118,12 +154,20 @@ class Config:
     def _parse_github_repositories(self):
         github = GitHub(self.github.access_token)
         for r in self.github.repositories.enum_repositories():
+            r = HostedRepo(r)
             yield Repo.from_github(github.get_repo(r), self)
+        for u in self.github.users.enum_users():
+            u = User(u)
+            yield from (Repo.from_github(r, self, u.dir) for r in github.get_user_repos(u))
 
     def _parse_bitbucket_repositories(self):
         bitbucket = Bitbucket(self.bitbucket.username, self.bitbucket.app_password)
         for r in self.bitbucket.repositories.enum_repositories():
+            r = HostedRepo(r)
             yield Repo.from_bitbucket(bitbucket.get_repo(r), self)
+        for u in self.bitbucket.users.enum_users():
+            u = User(u)
+            yield from (Repo.from_bitbucket(r, self, u.dir) for r in bitbucket.get_user_repos(u))
 
     def parse_repositories(self):
         yield from self._parse_explicit_repositories()
diff --git a/cgitize/github.py b/cgitize/github.py
index 2c88426..a5754af 100644
--- a/cgitize/github.py
+++ b/cgitize/github.py
@@ -13,11 +13,15 @@ class GitHub:
         self._impl = Github(access_token)
 
     def get_repo(self, repo):
-        if 'id' not in repo:
-            raise ValueError('every GitHub repository must have an ID')
-        repo_id = repo['id']
         try:
-            return self._impl.get_repo(repo_id)
+            return self._impl.get_repo(repo.id)
         except GithubException:
-            logging.error("Couldn't fetch repository: %s", repo_id)
+            logging.error("Couldn't fetch repository: %s", repo.id)
+            raise
+
+    def get_user_repos(self, user):
+        try:
+            return self._impl.get_user(user.name).get_repos()
+        except GithubException:
+            logging.error("Couldn't fetch user repositories: %s", user.name)
             raise
diff --git a/cgitize/repo.py b/cgitize/repo.py
index 99f39cf..cce3209 100644
--- a/cgitize/repo.py
+++ b/cgitize/repo.py
@@ -3,6 +3,8 @@
 # For details, see https://github.com/egor-tensin/cgitize.
 # Distributed under the MIT License.
 
+import os.path
+
 from cgitize.utils import url_remove_auth, url_replace_auth
 
 
@@ -10,18 +12,18 @@ class Repo:
     @staticmethod
     def from_config(src, config):
         if 'name' not in src:
-            raise ValueError('every repository must have a name')
+            raise ValueError("every repository must have 'name'")
         name = src['name']
         desc = src.get('desc')
         homepage = src.get('homepage')
         owner = src.get('owner', config.main.default_owner)
         if 'clone_url' not in src:
-            raise ValueError('every repository must have a clone URL')
+            raise ValueError("every repository must have 'clone_url'")
         clone_url = src['clone_url']
         return Repo(name, clone_url, owner=owner, desc=desc, homepage=homepage)
 
     @staticmethod
-    def from_github(src, config):
+    def from_github(src, config, subdir=None):
         name = src.name
         desc = src.description
         homepage = src.html_url
@@ -33,10 +35,10 @@ class Repo:
         url_auth = None if config.main.clone_via_ssh else config.github.url_auth
 
         return Repo(name, clone_url, owner=owner, desc=desc, homepage=homepage,
-                    url_auth=url_auth)
+                    url_auth=url_auth, subdir=subdir)
 
     @staticmethod
-    def from_bitbucket(src, config):
+    def from_bitbucket(src, config, subdir=None):
         name = src['name']
         desc = src['description']
         homepage = src['links']['html']['href']
@@ -56,16 +58,17 @@ class Repo:
         url_auth = None if config.main.clone_via_ssh else config.bitbucket.url_auth
 
         return Repo(name, clone_url, owner=owner, desc=desc, homepage=homepage,
-                    url_auth=url_auth)
+                    url_auth=url_auth, subdir=subdir)
 
     def __init__(self, name, clone_url, owner=None, desc=None, homepage=None,
-                 url_auth=None):
+                 url_auth=None, subdir=None):
         self._name = name
         self._desc = desc
         self._homepage = homepage
         self._owner = owner
         self._clone_url = clone_url
         self._url_auth = url_auth
+        self._dir = subdir
 
     @property
     def name(self):
@@ -102,3 +105,9 @@ class Repo:
         if not self.url_auth:
             return self.clone_url
         return url_replace_auth(self.clone_url, self.url_auth)
+
+    @property
+    def dir(self):
+        if self._dir is None:
+            return self.name
+        return os.path.join(self._dir, self.name)
diff --git a/examples/cgitize.toml b/examples/cgitize.toml
index b029328..19b7056 100644
--- a/examples/cgitize.toml
+++ b/examples/cgitize.toml
@@ -20,6 +20,12 @@ owner = "Your Name"
 # CGITIZE_GITHUB_ACCESS_TOKEN environment variable.
 #access_token = "XXX"
 
+[github.users.me]
+name = "egor-tensin"
+# To avoid directory name clashes with other repositories, you can specify a
+# subdirectory to put that user's repositories to.
+dir = "github-dir"
+
 # Some random repositories hosted on GitHub:
 [github.repositories.lens]
 id = "ekmett/lens"
@@ -36,6 +42,12 @@ id = "bobfang1992/pytomlpp"
 # environment variable).
 #username = "your-username"
 
+[bitbucket.users.me]
+name = "egor-tensin"
+# To avoid directory name clashes with other repositories, you can specify a
+# subdirectory to put that user's repositories to.
+dir = "bitbucket-dir"
+
 # Some random repositories hosted on Bitbucket:
 [bitbucket.repositories.cef]
 id = "chromiumembedded/cef"
diff --git a/test/integration/example/test.sh b/test/integration/example/test.sh
index 9345360..b8135f5 100755
--- a/test/integration/example/test.sh
+++ b/test/integration/example/test.sh
@@ -120,9 +120,16 @@ test_ssh() {
 
     setup_ssh
     cgitize
-    verify_repos lens cef wintun
+    verify_repos \
+        lens     \
+        cef      \
+        wintun   \
+        github-dir/cgitize-test-repository \
+        bitbucket-dir/cgitize-test-repository
     verify_origin lens 'git@github.com:ekmett/lens.git'
     verify_origin cef 'git@bitbucket.org:chromiumembedded/cef.git'
+    verify_origin github-dir/cgitize-test-repository 'git@github.com:egor-tensin/cgitize-test-repository.git'
+    verify_origin bitbucket-dir/cgitize-test-repository 'git@bitbucket.org:egor-tensin/cgitize-test-repository.git'
     cleanup
 }
 
@@ -134,9 +141,16 @@ test_https() {
 
     setup_https
     cgitize
-    verify_repos lens cef wintun
+    verify_repos \
+        lens     \
+        cef      \
+        wintun   \
+        github-dir/cgitize-test-repository \
+        bitbucket-dir/cgitize-test-repository
     verify_origin lens 'https://github.com/ekmett/lens.git'
     verify_origin cef 'https://bitbucket.org/chromiumembedded/cef.git'
+    verify_origin github-dir/cgitize-test-repository 'https://github.com/egor-tensin/cgitize-test-repository.git'
+    verify_origin bitbucket-dir/cgitize-test-repository 'https://bitbucket.org/egor-tensin/cgitize-test-repository.git'
     cleanup
 }