support mirroring all of user's repositories

5 files changed, 86 insertions, 18 deletions

diff --git a/cgitize/bitbucket.py b/cgitize/bitbucket.py
index 16f6d41..eaa8bb5 100644
--- a/cgitize/bitbucket.py
+++ b/cgitize/bitbucket.py
@@ -14,11 +14,22 @@ class Bitbucket:
         self._impl = Cloud(username=username, password=password, cloud=True)
 
     def get_repo(self, repo):
-        if 'id' not in repo:
-            raise ValueError('every Bitbucket repository must have an ID')
-        repo_id = repo['id']
         try:
-            return self._impl.repositories.get(repo_id)
+            return self._impl.repositories.get(repo.id)
         except HTTPError:
-            logging.error("Couldn't fetch repository: %s", repo_id)
+            logging.error("Couldn't fetch repository: %s", repo.id)
+            raise
+
+    def get_user_repos(self, user):
+        try:
+            page = 1
+            while True:
+                params = {'page': page}
+                response = self._impl.repositories.get(user.name, params=params)
+                yield from iter(response['values'])
+                if 'next' not in response:
+                    break
+                page += 1
+        except HTTPError:
+            logging.error("Couldn't fetch user repositories: %s", user.name)
             raise
diff --git a/cgitize/cgit.py b/cgitize/cgit.py
index 418f0c6..7f07876 100644
--- a/cgitize/cgit.py
+++ b/cgitize/cgit.py
@@ -96,7 +96,7 @@ class CGitRepositories:
         return abs_path
 
     def get_repo_dir(self, repo):
-        return os.path.join(self.dir, repo.name)
+        return os.path.join(self.dir, repo.dir)
 
     def update(self, repo):
         success = self._mirror_or_update(repo)
diff --git a/cgitize/config.py b/cgitize/config.py
index 8302f29..35ba05f 100644
--- a/cgitize/config.py
+++ b/cgitize/config.py
@@ -57,6 +57,7 @@ class MainSection(Section):
 class GitHubSection(Section):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self.users = UsersSection(self.impl.get('users', {}))
         self.repositories = RepositoriesSection(self.impl.get('repositories', {}))
 
     @property
@@ -71,6 +72,7 @@ class GitHubSection(Section):
 class BitbucketSection(Section):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self.users = UsersSection(self.impl.get('users', {}))
         self.repositories = RepositoriesSection(self.impl.get('repositories', {}))
 
     @property
@@ -89,12 +91,46 @@ class BitbucketSection(Section):
             return None
         return f'{username}:{password}'
 
+    def enum_repositories(self):
+        return map(HostedRepo, self.repositories.enum_repositories())
+
+
+class UsersSection(Section):
+    def enum_users(self):
+        return self.impl.values()
+
 
 class RepositoriesSection(Section):
     def enum_repositories(self):
         return self.impl.values()
 
 
+class User:
+    def __init__(self, impl):
+        if 'name' not in impl:
+            raise ValueError("every user must have 'name'")
+        self._impl = impl
+
+    @property
+    def name(self):
+        return self._impl['name']
+
+    @property
+    def dir(self):
+        return self._impl.get('dir')
+
+
+class HostedRepo:
+    def __init__(self, impl):
+        if 'id' not in impl:
+            raise ValueError("every hosted repository must have 'id'")
+        self._impl = impl
+
+    @property
+    def id(self):
+        return self._impl['id']
+
+
 class Config:
     DEFAULT_PATH = '/etc/cgitize/cgitize.toml'
 
@@ -118,12 +154,20 @@ class Config:
     def _parse_github_repositories(self):
         github = GitHub(self.github.access_token)
         for r in self.github.repositories.enum_repositories():
+            r = HostedRepo(r)
             yield Repo.from_github(github.get_repo(r), self)
+        for u in self.github.users.enum_users():
+            u = User(u)
+            yield from (Repo.from_github(r, self, u.dir) for r in github.get_user_repos(u))
 
     def _parse_bitbucket_repositories(self):
         bitbucket = Bitbucket(self.bitbucket.username, self.bitbucket.app_password)
         for r in self.bitbucket.repositories.enum_repositories():
+            r = HostedRepo(r)
             yield Repo.from_bitbucket(bitbucket.get_repo(r), self)
+        for u in self.bitbucket.users.enum_users():
+            u = User(u)
+            yield from (Repo.from_bitbucket(r, self, u.dir) for r in bitbucket.get_user_repos(u))
 
     def parse_repositories(self):
         yield from self._parse_explicit_repositories()
diff --git a/cgitize/github.py b/cgitize/github.py
index 2c88426..a5754af 100644
--- a/cgitize/github.py
+++ b/cgitize/github.py
@@ -13,11 +13,15 @@ class GitHub:
         self._impl = Github(access_token)
 
     def get_repo(self, repo):
-        if 'id' not in repo:
-            raise ValueError('every GitHub repository must have an ID')
-        repo_id = repo['id']
         try:
-            return self._impl.get_repo(repo_id)
+            return self._impl.get_repo(repo.id)
         except GithubException:
-            logging.error("Couldn't fetch repository: %s", repo_id)
+            logging.error("Couldn't fetch repository: %s", repo.id)
+            raise
+
+    def get_user_repos(self, user):
+        try:
+            return self._impl.get_user(user.name).get_repos()
+        except GithubException:
+            logging.error("Couldn't fetch user repositories: %s", user.name)
             raise
diff --git a/cgitize/repo.py b/cgitize/repo.py
index 99f39cf..cce3209 100644
--- a/cgitize/repo.py
+++ b/cgitize/repo.py
@@ -3,6 +3,8 @@
 # For details, see https://github.com/egor-tensin/cgitize.
 # Distributed under the MIT License.
 
+import os.path
+
 from cgitize.utils import url_remove_auth, url_replace_auth
 
 
@@ -10,18 +12,18 @@ class Repo:
     @staticmethod
     def from_config(src, config):
         if 'name' not in src:
-            raise ValueError('every repository must have a name')
+            raise ValueError("every repository must have 'name'")
         name = src['name']
         desc = src.get('desc')
         homepage = src.get('homepage')
         owner = src.get('owner', config.main.default_owner)
         if 'clone_url' not in src:
-            raise ValueError('every repository must have a clone URL')
+            raise ValueError("every repository must have 'clone_url'")
         clone_url = src['clone_url']
         return Repo(name, clone_url, owner=owner, desc=desc, homepage=homepage)
 
     @staticmethod
-    def from_github(src, config):
+    def from_github(src, config, subdir=None):
         name = src.name
         desc = src.description
         homepage = src.html_url
@@ -33,10 +35,10 @@ class Repo:
         url_auth = None if config.main.clone_via_ssh else config.github.url_auth
 
         return Repo(name, clone_url, owner=owner, desc=desc, homepage=homepage,
-                    url_auth=url_auth)
+                    url_auth=url_auth, subdir=subdir)
 
     @staticmethod
-    def from_bitbucket(src, config):
+    def from_bitbucket(src, config, subdir=None):
         name = src['name']
         desc = src['description']
         homepage = src['links']['html']['href']
@@ -56,16 +58,17 @@ class Repo:
         url_auth = None if config.main.clone_via_ssh else config.bitbucket.url_auth
 
         return Repo(name, clone_url, owner=owner, desc=desc, homepage=homepage,
-                    url_auth=url_auth)
+                    url_auth=url_auth, subdir=subdir)
 
     def __init__(self, name, clone_url, owner=None, desc=None, homepage=None,
-                 url_auth=None):
+                 url_auth=None, subdir=None):
         self._name = name
         self._desc = desc
         self._homepage = homepage
         self._owner = owner
         self._clone_url = clone_url
         self._url_auth = url_auth
+        self._dir = subdir
 
     @property
     def name(self):
@@ -102,3 +105,9 @@ class Repo:
         if not self.url_auth:
             return self.clone_url
         return url_replace_auth(self.clone_url, self.url_auth)
+
+    @property
+    def dir(self):
+        if self._dir is None:
+            return self.name
+        return os.path.join(self._dir, self.name)