factor various "repository" classes into Repo

Some other refactoring efforts are included.

9 files changed, 201 insertions, 196 deletions

diff --git a/.ci/docker/client/etc/cgitize.toml b/.ci/docker/client/etc/cgitize.toml
index bc46aca..4e8fda7 100644
--- a/.ci/docker/client/etc/cgitize.toml
+++ b/.ci/docker/client/etc/cgitize.toml
@@ -1,4 +1,4 @@
 [repositories.test_repo]
 
-id = "test_repo"
+name = "test_repo"
 clone_url = "root@server:~/test_repo"
diff --git a/.ci/local/test.sh b/.ci/local/test.sh
index 8826ff4..e78a4be 100755
--- a/.ci/local/test.sh
+++ b/.ci/local/test.sh
@@ -74,7 +74,7 @@ setup_cgitize_toml() {
 output = "$output_dir"
 
 [repositories.test_repo]
-id = "test_repo"
+name = "test_repo"
 clone_url = "$upstream_repo_dir"
 EOF
 }
diff --git a/cgitize/bitbucket.py b/cgitize/bitbucket.py
new file mode 100644
index 0000000..16f6d41
--- /dev/null
+++ b/cgitize/bitbucket.py
@@ -0,0 +1,24 @@
+# Copyright (c) 2021 Egor Tensin <Egor.Tensin@gmail.com>
+# This file is part of the "cgitize" project.
+# For details, see https://github.com/egor-tensin/cgitize.
+# Distributed under the MIT License.
+
+import logging
+
+from atlassian.bitbucket.cloud import Cloud
+from requests.exceptions import HTTPError
+
+
+class Bitbucket:
+    def __init__(self, username=None, password=None):
+        self._impl = Cloud(username=username, password=password, cloud=True)
+
+    def get_repo(self, repo):
+        if 'id' not in repo:
+            raise ValueError('every Bitbucket repository must have an ID')
+        repo_id = repo['id']
+        try:
+            return self._impl.repositories.get(repo_id)
+        except HTTPError:
+            logging.error("Couldn't fetch repository: %s", repo_id)
+            raise
diff --git a/cgitize/config.py b/cgitize/config.py
index 223dcd2..857d42a 100644
--- a/cgitize/config.py
+++ b/cgitize/config.py
@@ -9,7 +9,9 @@ import logging
 import os
 import sys
 
-from cgitize.repo import Repo, GitHub as GitHubRepo, Bitbucket as BitbucketRepo
+from cgitize.bitbucket import Bitbucket
+from cgitize.github import GitHub
+from cgitize.repo import Repo
 from cgitize.utils import chdir
 
 import tomli
@@ -28,13 +30,21 @@ class Section:
     def _get_config_path(self, *args, **kwargs):
         return os.path.abspath(self._get_config_value(*args, **kwargs))
 
+    def _get_config_or_env(self, key, env_name):
+        val = self._get_config_value(key, required=False)
+        if val is not None:
+            return val
+        if env_name in os.environ:
+            return os.environ[env_name]
+        return None
+
 
-class Main(Section):
+class MainSection(Section):
     DEFAULT_OUTPUT_DIR = '/var/tmp/cgitize/output'
 
     @property
     def output(self):
-        return self._get_config_path('output', default=Main.DEFAULT_OUTPUT_DIR)
+        return self._get_config_path('output', default=MainSection.DEFAULT_OUTPUT_DIR)
 
     @property
     def clone_url(self):
@@ -49,52 +59,51 @@ class Main(Section):
         return self._get_config_value('ssh', default=True)
 
 
-class GitHub(Section):
+class GitHubSection(Section):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.repositories = Repositories(self.impl.get('repositories', {}), GitHubRepo)
+        self.repositories = RepositoriesSection(self.impl.get('repositories', {}))
 
     @property
     def access_token(self):
-        access_token = self._get_config_value('access_token', required=False)
-        if access_token is not None:
-            return access_token
-        env_var = 'CGITIZE_GITHUB_ACCESS_TOKEN'
-        if env_var in os.environ:
-            return os.environ[env_var]
-        return None
+        return self._get_config_or_env('access_token', 'CGITIZE_GITHUB_ACCESS_TOKEN')
+
+    @property
+    def url_auth(self):
+        return self.access_token
 
     def enum_repositories(self):
         return self.repositories.enum_repositories()
 
 
-class Bitbucket(Section):
+class BitbucketSection(Section):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.repositories = Repositories(self.impl.get('repositories', {}), BitbucketRepo)
+        self.repositories = RepositoriesSection(self.impl.get('repositories', {}))
 
     @property
     def app_password(self):
-        app_password = self._get_config_value('app_password', required=False)
-        if app_password is not None:
-            return app_password
-        env_var = 'CGITIZE_BITBUCKET_APP_PASSWORD'
-        if env_var in os.environ:
-            return os.environ[env_var]
-        return None
+        return self._get_config_or_env('app_password', 'CGITIZE_BITBUCKET_APP_PASSWORD')
+
+    @property
+    def username(self):
+        return self._get_config_or_env('username', 'CGITIZE_BITBUCKET_USERNAME')
+
+    @property
+    def url_auth(self):
+        username = self.username
+        password = self.app_password
+        if username is None or password is None:
+            return None
+        return f'{username}:{password}'
 
     def enum_repositories(self):
         return self.repositories.enum_repositories()
 
 
-class Repositories(Section):
-    def __init__(self, impl, repo_cls=Repo):
-        super().__init__(impl)
-        self.repo_cls = repo_cls
-
+class RepositoriesSection(Section):
     def enum_repositories(self):
-        for k, v in self.impl.items():
-            yield self.repo_cls.from_config(v)
+        return self.impl.values()
 
 
 class Config:
@@ -108,12 +117,26 @@ class Config:
         self.path = os.path.abspath(path)
         with open(self.path, 'rb') as f:
             self.impl = tomli.load(f)
-        self.main = Main(self.impl)
-        self.repositories = Repositories(self.impl.get('repositories', {}))
-        self.github = GitHub(self.impl.get('github', {}))
-        self.bitbucket = Bitbucket(self.impl.get('bitbucket', {}))
-
-    def enum_repositories(self):
-        yield from self.repositories.enum_repositories()
-        yield from self.github.enum_repositories()
-        yield from self.bitbucket.enum_repositories()
+        self.main = MainSection(self.impl)
+        self.repositories = RepositoriesSection(self.impl.get('repositories', {}))
+        self.github = GitHubSection(self.impl.get('github', {}))
+        self.bitbucket = BitbucketSection(self.impl.get('bitbucket', {}))
+
+    def _parse_explicit_repositories(self):
+        for r in self.repositories.enum_repositories():
+            yield Repo.from_config(r, self)
+
+    def _parse_github_repositories(self):
+        github = GitHub(self.github.access_token)
+        for r in self.github.repositories.enum_repositories():
+            yield Repo.from_github(github.get_repo(r), self)
+
+    def _parse_bitbucket_repositories(self):
+        bitbucket = Bitbucket(self.bitbucket.username, self.bitbucket.app_password)
+        for r in self.bitbucket.repositories.enum_repositories():
+            yield Repo.from_bitbucket(bitbucket.get_repo(r), self)
+
+    def parse_repositories(self):
+        yield from self._parse_explicit_repositories()
+        yield from self._parse_github_repositories()
+        yield from self._parse_bitbucket_repositories()
diff --git a/cgitize/github.py b/cgitize/github.py
new file mode 100644
index 0000000..2c88426
--- /dev/null
+++ b/cgitize/github.py
@@ -0,0 +1,23 @@
+# Copyright (c) 2021 Egor Tensin <Egor.Tensin@gmail.com>
+# This file is part of the "cgitize" project.
+# For details, see https://github.com/egor-tensin/cgitize.
+# Distributed under the MIT License.
+
+import logging
+
+from github import Github, GithubException
+
+
+class GitHub:
+    def __init__(self, access_token):
+        self._impl = Github(access_token)
+
+    def get_repo(self, repo):
+        if 'id' not in repo:
+            raise ValueError('every GitHub repository must have an ID')
+        repo_id = repo['id']
+        try:
+            return self._impl.get_repo(repo_id)
+        except GithubException:
+            logging.error("Couldn't fetch repository: %s", repo_id)
+            raise
diff --git a/cgitize/main.py b/cgitize/main.py
index 6b4705f..e7d2dfd 100644
--- a/cgitize/main.py
+++ b/cgitize/main.py
@@ -36,10 +36,8 @@ def main(args=None):
         cgit_server = CGitServer(config.main.clone_url)
         output = CGitRepositories(config.main.output, cgit_server, force=args.force)
         success = True
-        for repo in config.enum_repositories():
+        for repo in config.parse_repositories():
             if args.repos is None or repo.repo_id in args.repos:
-                repo.fill_defaults(config)
-                repo.validate()
                 if not output.update(repo):
                     success = False
         if success:
diff --git a/cgitize/repo.py b/cgitize/repo.py
index 01b7430..4c91968 100644
--- a/cgitize/repo.py
+++ b/cgitize/repo.py
@@ -5,47 +5,76 @@
 
 import abc
 import os.path
-from urllib.parse import urlsplit, urlunsplit
+
+from cgitize.utils import url_remove_auth, url_replace_auth
 
 
 class Repo:
-    @classmethod
-    def from_config(cls, cfg):
-        if 'id' not in cfg:
-            raise ValueError('every repository must have its id defined')
-        return cls(cfg['id'], clone_url=cfg.get('clone_url'),
-                   owner=cfg.get('owner'), desc=cfg.get('desc'),
-                   homepage=cfg.get('homepage'))
-
-    def __init__(self, name, clone_url=None, owner=None, desc=None,
-                 homepage=None):
+    @staticmethod
+    def from_config(src, config):
+        if 'name' not in src:
+            raise ValueError('every repository must have a name')
+        name = src['name']
+        desc = src.get('desc')
+        homepage = src.get('homepage')
+        owner = src.get('owner', config.main.default_owner)
+        if 'clone_url' not in src:
+            raise ValueError('every repository must have a clone URL')
+        clone_url = src['clone_url']
+        return Repo(name, clone_url, owner=owner, desc=desc, homepage=homepage)
+
+    @staticmethod
+    def from_github(src, config):
+        name = src.name
+        desc = src.description
+        homepage = src.html_url
+        owner = src.owner.name
+
+        https_url = src.clone_url
+        ssh_url = src.ssh_url
+        clone_url = ssh_url if config.main.via_ssh else https_url
+        url_auth = None if config.main.via_ssh else config.github.url_auth
+
+        return Repo(name, clone_url, owner=owner, desc=desc, homepage=homepage,
+                    url_auth=url_auth)
+
+    @staticmethod
+    def from_bitbucket(src, config):
+        name = src['name']
+        desc = src['description']
+        homepage = src['links']['html']['href']
+        owner = src['owner']['display_name']
+
+        https_urls = [link for link in src['links']['clone'] if link['name'] == 'https']
+        if len(https_urls) != 1:
+            raise RuntimeError(f"no https:// clone URL for repository '{name}'?!")
+        # Bitbucket leaves the username in the URL... Sigh.
+        https_url = url_remove_auth(https_urls[0]['href'])
+
+        ssh_urls = [link for link in src['links']['clone'] if link ['name'] == 'ssh']
+        if len(ssh_urls) != 1: raise RuntimeError(f"no ssh:// clone URL for repository '{name}'?!")
+        ssh_url = ssh_urls[0]['href']
+
+        clone_url = ssh_url if config.main.via_ssh else https_url
+        url_auth = None if config.main.via_ssh else config.bitbucket.url_auth
+
+        return Repo(name, clone_url, owner=owner, desc=desc, homepage=homepage,
+                    url_auth=url_auth)
+
+    def __init__(self, name, clone_url, owner=None, desc=None, homepage=None,
+                 url_auth=None):
         self._name = name
-        self._clone_url = clone_url
-        self._owner = owner
         self._desc = desc
         self._homepage = homepage
-
-    def fill_defaults(self, config):
-        if self._owner is None:
-            self._owner = config.main.default_owner
-
-    def validate(self):
-        if self.clone_url is None:
-            raise RuntimeError('upstream repository URL must be specified')
+        self._owner = owner
+        self._clone_url = clone_url
+        self._url_auth = url_auth
 
     @property
     def name(self):
         return self._name
 
     @property
-    def clone_url(self):
-        return self._clone_url
-
-    @property
-    def owner(self):
-        return self._owner
-
-    @property
     def desc(self):
         if self._desc is not None:
             return self._desc
@@ -60,135 +89,19 @@ class Repo:
         return self._homepage
 
     @property
-    def url_auth(self):
-        return False
-
-
-class HostedRepo(Repo, abc.ABC):
-    @classmethod
-    def from_config(cls, cfg):
-        if 'id' not in cfg:
-            raise ValueError('every repository must have its id defined')
-        return cls(cfg['id'], owner=cfg.get('owner'), desc=cfg.get('desc'),
-                   homepage=cfg.get('homepage'))
-
-    @staticmethod
-    def split_repo_id(repo_id):
-        components = repo_id.split('/')
-        if len(components) != 2:
-            raise ValueError(f'repository ID must be in the USER/NAME format: {repo_id}')
-        user, name = components
-        return user, name
-
-    def __init__(self, repo_id, owner=None, desc=None, homepage=None,
-                 via_ssh=True):
-        user, name = self.split_repo_id(repo_id)
-        super().__init__(name, clone_url=None, owner=owner, desc=desc,
-                         homepage=homepage)
-        self._user = user
-        self._via_ssh = via_ssh
-
-    def fill_defaults(self, config):
-        super().fill_defaults(config)
-        self._via_ssh = config.main.via_ssh
-
-    @property
-    def user(self):
-        return self._user
-
-    @property
-    @abc.abstractmethod
-    def provider_name(self):
-        pass
-
-    @property
-    @abc.abstractmethod
-    def clone_url_ssh(self):
-        pass
-
-    @property
-    @abc.abstractmethod
-    def clone_url_https(self):
-        pass
+    def owner(self):
+        return self._owner
 
     @property
     def clone_url(self):
-        if self._via_ssh:
-            return self.clone_url_ssh
-        return self.clone_url_https
-
-    @property
-    def clone_url_with_auth(self):
-        if self._via_ssh:
-            return self.clone_url_ssh
-        auth = self.url_auth
-        clone_url = self.clone_url_https
-        if not auth:
-            return clone_url
-        clone_url = urlsplit(clone_url)
-        clone_url = clone_url._replace(netloc=f'{auth}@{clone_url.netloc}')
-        return urlunsplit(clone_url)
-
-
-class GitHub(HostedRepo):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._access_token = None
-
-    def fill_defaults(self, config):
-        super().fill_defaults(config)
-        self._access_token = config.github.access_token
-
-    @property
-    def provider_name(self):
-        return 'GitHub'
-
-    @property
-    def homepage(self):
-        return f'https://github.com/{self.user}/{self.name}'
-
-    @property
-    def url_auth(self):
-        if self._access_token is None:
-            return ''
-        return f'{self._access_token}'
-
-    @property
-    def clone_url_ssh(self):
-        return f'ssh://git@github.com/{self.user}/{self.name}.git'
-
-    @property
-    def clone_url_https(self):
-        return f'https://github.com/{self.user}/{self.name}.git'
-
-
-class Bitbucket(HostedRepo):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._app_password = None
-
-    def fill_defaults(self, config):
-        super().fill_defaults(config)
-        self._app_password = config.bitbucket.app_password
-
-    @property
-    def provider_name(self):
-        return 'Bitbucket'
-
-    @property
-    def homepage(self):
-        return f'https://bitbucket.org/{self.user}/{self.name.lower()}'
+        return self._clone_url
 
     @property
     def url_auth(self):
-        if self._app_password is None:
-            return ''
-        return f'{self.user}:{self._app_password}'
+        return self._url_auth
 
     @property
-    def clone_url_ssh(self):
-        return f'ssh://git@bitbucket.org/{self.user}/{self.name}.git'
-
-    @property
-    def clone_url_https(self):
-        return f'https://bitbucket.org/{self.user}/{self.name}.git'
+    def clone_url_with_auth(self):
+        if not self.url_auth:
+            return self.clone_url
+        return url_replace_auth(self.clone_url, self.url_auth)
diff --git a/cgitize/utils.py b/cgitize/utils.py
index 2f91939..64b419a 100644
--- a/cgitize/utils.py
+++ b/cgitize/utils.py
@@ -9,6 +9,7 @@ import os
 import stat
 import subprocess
 import sys
+from urllib.parse import urlsplit, urlunsplit
 
 
 @contextmanager
@@ -92,3 +93,22 @@ def protected_file(path):
             yield
         finally:
             os.unlink(path)
+
+
+def url_replace_auth(url, username, password=None):
+    parts = urlsplit(url)
+    netloc = username
+    if password is not None:
+        netloc += f':{password}'
+    netloc += f'@{parts.hostname}'
+    if parts.port is not None:
+        netloc += f':{parts.port}'
+    parts = parts._replace(netloc=netloc)
+    return urlunsplit(parts)
+
+
+def url_remove_auth(url):
+    parts = urlsplit(url)
+    netloc = parts.hostname
+    parts = parts._replace(netloc=netloc)
+    return urlunsplit(parts)
diff --git a/examples/cgitize.toml b/examples/cgitize.toml
index 6046e36..fcf6b92 100644
--- a/examples/cgitize.toml
+++ b/examples/cgitize.toml
@@ -31,6 +31,10 @@ id = "bobfang1992/pytomlpp"
 # passwords" to access them. Also can be provided using the
 # CGITIZE_BITBUCKET_APP_PASSWORD environment variable.
 #app_password = "XXX"
+# Contrary to GitHub, Bitbucket doesn't associate app passwords with a
+# username, so you also need that (CGITIZE_BITBUCKET_USERNAME is the
+# environment variable).
+#username = "your-username"
 
 # Some random repositories hosted on Bitbucket:
 [bitbucket.repositories.cef]
@@ -42,7 +46,7 @@ id = "berkeleylab/upc-runtime"
 
 # Some random repositories hosted on the web:
 [repositories.wintun]
-id = "wintun"
+name = "wintun"
 clone_url = "https://git.zx2c4.com/wintun"
 owner = "Jason A. Donenfeld"
 desc = "Layer 3 TUN Driver for Windows"