merge my_repos.py to the config

The config is also in the TOML format now. It's a bit messy for the
moment, but I'll fix it.

15 files changed, 198 insertions, 190 deletions

diff --git a/.ci/docker/client/etc/cgitize.toml b/.ci/docker/client/etc/cgitize.toml
new file mode 100644
index 0000000..bc46aca
--- /dev/null
+++ b/.ci/docker/client/etc/cgitize.toml
@@ -0,0 +1,4 @@
+[repositories.test_repo]
+
+id = "test_repo"
+clone_url = "root@server:~/test_repo"
diff --git a/.ci/docker/client/etc/my_repos.py b/.ci/docker/client/etc/my_repos.py
deleted file mode 100644
index 58cf542..0000000
--- a/.ci/docker/client/etc/my_repos.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from cgitize.repo import Repo
-
-
-MY_REPOS = (
-    Repo('test_repo', clone_url='root@server:~/test_repo'),
-)
diff --git a/.ci/local/test.sh b/.ci/local/test.sh
index 8ae7fe9..8826ff4 100755
--- a/.ci/local/test.sh
+++ b/.ci/local/test.sh
@@ -8,8 +8,7 @@ readonly script_dir
 
 upstream_repo_dir=
 readonly etc_dir="$script_dir/etc"
-readonly cgitize_conf_path="$etc_dir/cgitize.conf"
-readonly my_repos_path="$etc_dir/my_repos.py"
+readonly cgitize_toml_path="$etc_dir/cgitize.toml"
 readonly output_dir="$script_dir/output"
 
 cleanup() {
@@ -61,47 +60,27 @@ add_commits() {
     popd > /dev/null
 }
 
-setup_cgitize_conf() {
+setup_cgitize_toml() {
     echo
     echo ----------------------------------------------------------------------
-    echo cgitize.conf
+    echo cgitize.toml
     echo ----------------------------------------------------------------------
 
     local conf_dir
-    conf_dir="$( dirname -- "$cgitize_conf_path" )"
+    conf_dir="$( dirname -- "$cgitize_toml_path" )"
     mkdir -p -- "$conf_dir"
 
-    cat <<EOF | tee "$cgitize_conf_path"
-[DEFAULT]
+    cat <<EOF | tee "$cgitize_toml_path"
+output = "$output_dir"
 
-my_repos = $( basename -- "$my_repos_path" )
-output = $output_dir
-EOF
-}
-
-setup_my_repos_py() {
-    echo
-    echo ----------------------------------------------------------------------
-    echo my_repos.py
-    echo ----------------------------------------------------------------------
-
-    local conf_dir
-    conf_dir="$( dirname -- "$my_repos_path" )"
-    mkdir -p -- "$conf_dir"
-
-    cat <<EOF | tee "$my_repos_path"
-from cgitize.repo import Repo
-
-
-MY_REPOS = (
-    Repo('test_repo', clone_url='$upstream_repo_dir'),
-)
+[repositories.test_repo]
+id = "test_repo"
+clone_url = "$upstream_repo_dir"
 EOF
 }
 
 setup_cgitize() {
-    setup_cgitize_conf
-    setup_my_repos_py
+    setup_cgitize_toml
 }
 
 setup_bare() {
@@ -127,7 +106,7 @@ cgitize() {
     echo Running cgitize
     echo ----------------------------------------------------------------------
 
-    python3 -m cgitize.main --config "$cgitize_conf_path" --verbose
+    python3 -m cgitize.main --config "$cgitize_toml_path" --verbose
 }
 
 check_contains() {
diff --git a/.dockerignore b/.dockerignore
index eabf2d8..8f869af 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,3 +2,4 @@
 
 !/cgitize/**
 !/docker/**
+!/requirements.txt
diff --git a/Dockerfile b/Dockerfile
index b314d0f..5378dd1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,12 +1,17 @@
 FROM alpine:3.13
 
-RUN apk add --no-cache bash git openssh-client python3 tini
+RUN build_deps='gcc libffi-dev make musl-dev python3-dev py3-pip' && \
+    runtime_deps='bash git openssh-client python3 tini' && \
+    apk add --no-cache $build_deps $runtime_deps
 
 ARG ssh_sock_dir=/var/run/cgitize
 ARG ssh_sock_path="$ssh_sock_dir/ssh-agent.sock"
 
 ENV SSH_AUTH_SOCK "$ssh_sock_path"
 
+COPY ["requirements.txt", "/tmp/"]
+RUN pip3 install -r /tmp/requirements.txt
+
 COPY ["docker/entrypoint.sh", "/"]
 COPY ["cgitize/", "/usr/src/cgitize/"]
 WORKDIR /usr/src
diff --git a/README.md b/README.md
index ee493b5..a74e4c0 100644
--- a/README.md
+++ b/README.md
@@ -15,28 +15,23 @@ Installation
 Usage
 -----
 
-cgitize uses two config files.
+Pass the path to the config to `cgitize` (/etc/cgitize/cgitize.toml by
+default):
 
-* cgitize.conf contains all the settings; see example at
-[examples/cgitize.conf].
-* my_repos.py contains the list of repositories to mirror; see example at
-[examples/my_repos.py].
+    cgitize --config path/to/cgitize.toml
 
-Pass the path to cgitize.conf using the `--config` parameter:
-
-    cgitize --config path/to/cgitize.conf
+See an example config file at [examples/cgitize.toml].
 
 cgitize uses the `git` executable, which might use `ssh` internally.
-Make sure the required keys are loaded to a ssh-agent (or use authentication
-tokens).
+Make sure the required keys are loaded to a ssh-agent (or use access
+tokens/application passwords).
 
-[examples/cgitize.conf]: examples/cgitize.conf
-[examples/my_repos.py]: examples/my_repos.py
+[examples/cgitize.toml]: examples/cgitize.toml
 
 ### Docker
 
 The image is **egortensin/cgitize**.
-The container reads the config from /etc/cgitize/cgitize.conf and writes the
+The container reads the config from /etc/cgitize/cgitize.toml and writes the
 repositories to /var/tmp/cgitize/output.
 If SSH is required, the socket should be mapped to
 /var/run/cgitize/ssh-agent.sock.
diff --git a/cgitize/cgit.py b/cgitize/cgit.py
index fe715d5..53a726a 100644
--- a/cgitize/cgit.py
+++ b/cgitize/cgit.py
@@ -18,7 +18,7 @@ class CGitServer:
     def get_clone_url(self, repo):
         if self.clone_url is None:
             return None
-        return self.clone_url.format(repo_id=repo.repo_id)
+        return self.clone_url.format(name=repo.name)
 
 
 class CGitRCWriter:
@@ -96,7 +96,7 @@ class CGitRepositories:
         return abs_path
 
     def get_repo_dir(self, repo):
-        return os.path.join(self.dir, repo.repo_id)
+        return os.path.join(self.dir, repo.name)
 
     def update(self, repo):
         success = self._mirror_or_update(repo)
@@ -130,7 +130,7 @@ class CGitRepositories:
                 # Jeez, there's a proper local repository in the target
                 # directory already with a different upstream; something's
                 # wrong, fix it manually.
-                logging.warning("Existing repository '%s' doesn't match the specified clone URL: %s", repo.repo_id, repo.clone_url)
+                logging.warning("Existing repository '%s' doesn't match the specified clone URL: %s", repo.name, repo.clone_url)
                 if self.force:
                     # Unless --force was specified, in which case we overwrite
                     # the repository.
@@ -142,7 +142,7 @@ class CGitRepositories:
             return self._update_existing(repo)
 
     def _mirror(self, repo):
-        logging.info("Mirroring repository '%s' from: %s", repo.repo_id, repo.clone_url)
+        logging.info("Mirroring repository '%s' from: %s", repo.name, repo.clone_url)
         repo_dir = self.get_repo_dir(repo)
         if os.path.isdir(repo_dir):
             try:
@@ -159,7 +159,7 @@ class CGitRepositories:
             return Git.check('remote', 'set-url', 'origin', repo.clone_url)
 
     def _update_existing(self, repo):
-        logging.info("Updating repository '%s'", repo.repo_id)
+        logging.info("Updating repository '%s'", repo.name)
         repo_dir = self.get_repo_dir(repo)
         with chdir(repo_dir):
             with Git.setup_auth(repo):
diff --git a/cgitize/config.py b/cgitize/config.py
index a630d7a..8171860 100644
--- a/cgitize/config.py
+++ b/cgitize/config.py
@@ -9,73 +9,99 @@ import logging
 import os.path
 import sys
 
+from cgitize.repo import Repo, GitHub as GitHubRepo, Bitbucket as BitbucketRepo
 from cgitize.utils import chdir
 
+import tomli
 
-class Config:
-    DEFAULT_PATH = '/etc/cgitize/cgitize.conf'
-    DEFAULT_OUTPUT_DIR = '/var/tmp/cgitize/output'
-    DEFAULT_MY_REPOS_PATH = '/etc/cgitize/my_repos.py'
 
-    @staticmethod
-    def read(path):
-        return Config(path)
+class Section:
+    def __init__(self, impl):
+        self.impl = impl
 
-    def __init__(self, path):
-        self.path = os.path.abspath(path)
-        self.impl = configparser.ConfigParser()
-        self.impl.read(path)
+    def _get_config_value(self, key, required=True, default=None):
+        if required and default is None:
+            if not key in self.impl:
+                raise RuntimeError(f'configuration value is missing: {key}')
+        return self.impl.get(key, default)
+
+    def _get_config_path(self, *args, **kwargs):
+        return os.path.abspath(self._get_config_value(*args, **kwargs))
 
-    def _resolve_relative(self, path):
-        if os.path.isabs(path):
-            return path
-        with chdir(os.path.dirname(self.path)):
-            path = os.path.abspath(path)
-            return path
+
+class Main(Section):
+    DEFAULT_OUTPUT_DIR = '/var/tmp/cgitize/output'
 
     @property
     def output(self):
-        path = self.impl.get('DEFAULT', 'output', fallback=Config.DEFAULT_OUTPUT_DIR)
-        return self._resolve_relative(path)
+        return self._get_config_path('output', default=Main.DEFAULT_OUTPUT_DIR)
 
     @property
     def clone_url(self):
-        return self.impl.get('DEFAULT', 'clone_url', fallback=None)
+        return self._get_config_value('clone_url', required=False)
 
     @property
     def default_owner(self):
-        return self.impl.get('DEFAULT', 'owner', fallback=None)
+        return self._get_config_value('owner', required=False)
 
     @property
     def via_ssh(self):
-        return self.impl.getboolean('DEFAULT', 'ssh', fallback=True)
+        return self._get_config_value('ssh', default=True)
 
-    @property
-    def github_username(self):
-        return self.impl.get('GITHUB', 'username', fallback=None)
 
-    @property
-    def github_access_token(self):
-        return self.impl.get('GITHUB', 'access_token', fallback=None)
+class GitHub(Section):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.repositories = Repositories(self.impl.get('repositories', {}), GitHubRepo)
 
     @property
-    def bitbucket_username(self):
-        return self.impl.get('BITBUCKET', 'username', fallback=None)
+    def access_token(self):
+        return self._get_config_value('access_token', required=False)
 
-    @property
-    def bitbucket_app_password(self):
-        return self.impl.get('BITBUCKET', 'app_password', fallback=None)
+    def enum_repositories(self):
+        return self.repositories.enum_repositories()
+
+
+class Bitbucket(Section):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.repositories = Repositories(self.impl.get('repositories', {}), BitbucketRepo)
 
     @property
-    def my_repos(self):
-        path = self.impl.get('DEFAULT', 'my_repos', fallback=Config.DEFAULT_MY_REPOS_PATH)
-        return self._resolve_relative(path)
-
-    def import_my_repos(self):
-        sys.path.append(os.path.dirname(self.my_repos))
-        if not os.path.exists(self.my_repos):
-            logging.error("Couldn't find my_repos.py at: %s", self.my_repos)
-            return None
-        module_name = os.path.splitext(os.path.basename(self.my_repos))[0]
-        module = importlib.import_module(module_name)
-        return module.MY_REPOS
+    def app_password(self):
+        return self._get_config_value('app_password', required=False)
+
+    def enum_repositories(self):
+        return self.repositories.enum_repositories()
+
+
+class Repositories(Section):
+    def __init__(self, impl, repo_cls=Repo):
+        super().__init__(impl)
+        self.repo_cls = repo_cls
+
+    def enum_repositories(self):
+        for k, v in self.impl.items():
+            yield self.repo_cls.from_config(v)
+
+
+class Config:
+    DEFAULT_PATH = '/etc/cgitize/cgitize.toml'
+
+    @staticmethod
+    def read(path):
+        return Config(path)
+
+    def __init__(self, path):
+        self.path = os.path.abspath(path)
+        with open(self.path, 'rb') as f:
+            self.impl = tomli.load(f)
+        self.main = Main(self.impl)
+        self.repositories = Repositories(self.impl.get('repositories', {}))
+        self.github = GitHub(self.impl.get('github', {}))
+        self.bitbucket = Bitbucket(self.impl.get('bitbucket', {}))
+
+    def enum_repositories(self):
+        yield from self.repositories.enum_repositories()
+        yield from self.github.enum_repositories()
+        yield from self.bitbucket.enum_repositories()
diff --git a/cgitize/main.py b/cgitize/main.py
index 0c8f029..6b4705f 100644
--- a/cgitize/main.py
+++ b/cgitize/main.py
@@ -33,13 +33,10 @@ def main(args=None):
     args = parse_args()
     with setup_logging(args.verbose):
         config = Config.read(args.config)
-        my_repos = config.import_my_repos()
-        if my_repos is None:
-            return 1
-        cgit_server = CGitServer(config.clone_url)
-        output = CGitRepositories(config.output, cgit_server, force=args.force)
+        cgit_server = CGitServer(config.main.clone_url)
+        output = CGitRepositories(config.main.output, cgit_server, force=args.force)
         success = True
-        for repo in my_repos:
+        for repo in config.enum_repositories():
             if args.repos is None or repo.repo_id in args.repos:
                 repo.fill_defaults(config)
                 repo.validate()
diff --git a/cgitize/repo.py b/cgitize/repo.py
index 8ecbcf5..01b7430 100644
--- a/cgitize/repo.py
+++ b/cgitize/repo.py
@@ -9,14 +9,17 @@ from urllib.parse import urlsplit, urlunsplit
 
 
 class Repo:
-    @staticmethod
-    def extract_repo_name(repo_id):
-        return os.path.basename(repo_id)
-
-    def __init__(self, repo_id, clone_url=None, owner=None, desc=None,
+    @classmethod
+    def from_config(cls, cfg):
+        if 'id' not in cfg:
+            raise ValueError('every repository must have its id defined')
+        return cls(cfg['id'], clone_url=cfg.get('clone_url'),
+                   owner=cfg.get('owner'), desc=cfg.get('desc'),
+                   homepage=cfg.get('homepage'))
+
+    def __init__(self, name, clone_url=None, owner=None, desc=None,
                  homepage=None):
-        self._repo_id = repo_id
-        self._repo_name = self.extract_repo_name(repo_id)
+        self._name = name
         self._clone_url = clone_url
         self._owner = owner
         self._desc = desc
@@ -24,19 +27,15 @@ class Repo:
 
     def fill_defaults(self, config):
         if self._owner is None:
-            self._owner = config.default_owner
+            self._owner = config.main.default_owner
 
     def validate(self):
         if self.clone_url is None:
             raise RuntimeError('upstream repository URL must be specified')
 
     @property
-    def repo_id(self):
-        return self._repo_id
-
-    @property
-    def repo_name(self):
-        return self._repo_name
+    def name(self):
+        return self._name
 
     @property
     def clone_url(self):
@@ -54,7 +53,7 @@ class Repo:
             return self.homepage
         if self.clone_url:
             return self.clone_url
-        return self.repo_name
+        return self.name
 
     @property
     def homepage(self):
@@ -66,21 +65,32 @@ class Repo:
 
 
 class HostedRepo(Repo, abc.ABC):
+    @classmethod
+    def from_config(cls, cfg):
+        if 'id' not in cfg:
+            raise ValueError('every repository must have its id defined')
+        return cls(cfg['id'], owner=cfg.get('owner'), desc=cfg.get('desc'),
+                   homepage=cfg.get('homepage'))
+
+    @staticmethod
+    def split_repo_id(repo_id):
+        components = repo_id.split('/')
+        if len(components) != 2:
+            raise ValueError(f'repository ID must be in the USER/NAME format: {repo_id}')
+        user, name = components
+        return user, name
+
     def __init__(self, repo_id, owner=None, desc=None, homepage=None,
-                 user=None, via_ssh=True):
-        super().__init__(repo_id, clone_url=None, owner=owner, desc=desc,
+                 via_ssh=True):
+        user, name = self.split_repo_id(repo_id)
+        super().__init__(name, clone_url=None, owner=owner, desc=desc,
                          homepage=homepage)
         self._user = user
         self._via_ssh = via_ssh
 
     def fill_defaults(self, config):
         super().fill_defaults(config)
-        self._via_ssh = config.via_ssh
-
-    def validate(self):
-        super().validate()
-        if self.user is None:
-            raise RuntimeError(f'neither explicit or default {self.provider_name} username was specified')
+        self._via_ssh = config.main.via_ssh
 
     @property
     def user(self):
@@ -127,9 +137,7 @@ class GitHub(HostedRepo):
 
     def fill_defaults(self, config):
         super().fill_defaults(config)
-        if self._user is None:
-            self._user = config.github_username
-        self._access_token = config.github_access_token
+        self._access_token = config.github.access_token
 
     @property
     def provider_name(self):
@@ -137,7 +145,7 @@ class GitHub(HostedRepo):
 
     @property
     def homepage(self):
-        return f'https://github.com/{self.user}/{self.repo_name}'
+        return f'https://github.com/{self.user}/{self.name}'
 
     @property
     def url_auth(self):
@@ -147,11 +155,11 @@ class GitHub(HostedRepo):
 
     @property
     def clone_url_ssh(self):
-        return f'ssh://git@github.com/{self.user}/{self.repo_name}.git'
+        return f'ssh://git@github.com/{self.user}/{self.name}.git'
 
     @property
     def clone_url_https(self):
-        return f'https://github.com/{self.user}/{self.repo_name}.git'
+        return f'https://github.com/{self.user}/{self.name}.git'
 
 
 class Bitbucket(HostedRepo):
@@ -161,9 +169,7 @@ class Bitbucket(HostedRepo):
 
     def fill_defaults(self, config):
         super().fill_defaults(config)
-        if self._user is None:
-            self._user = config.bitbucket_username
-        self._app_password = config.bitbucket_app_password
+        self._app_password = config.bitbucket.app_password
 
     @property
     def provider_name(self):
@@ -171,7 +177,7 @@ class Bitbucket(HostedRepo):
 
     @property
     def homepage(self):
-        return f'https://bitbucket.org/{self.user}/{self.repo_name.lower()}'
+        return f'https://bitbucket.org/{self.user}/{self.name.lower()}'
 
     @property
     def url_auth(self):
@@ -181,8 +187,8 @@ class Bitbucket(HostedRepo):
 
     @property
     def clone_url_ssh(self):
-        return f'ssh://git@bitbucket.org/{self.user}/{self.repo_name}.git'
+        return f'ssh://git@bitbucket.org/{self.user}/{self.name}.git'
 
     @property
     def clone_url_https(self):
-        return f'https://bitbucket.org/{self.user}/{self.repo_name}.git'
+        return f'https://bitbucket.org/{self.user}/{self.name}.git'
diff --git a/examples/cgitize.conf b/examples/cgitize.conf
deleted file mode 100644
index a85daee..0000000
--- a/examples/cgitize.conf
+++ /dev/null
@@ -1,34 +0,0 @@
-# All settings are optional.
-
-[DEFAULT]
-
-# /etc/cgitize/my_repos.py by default.
-my_repos = /path/to/my_repos.py
-
-# /var/tmp/cgitize/output by default.
-output = /path/to/output/
-
-# URL to clone from the output directory. {repo_id} is replaced by the
-# repository ID (in the NAME or SECTION/NAME format).
-clone_url = http://example.com:8080/git/{repo_id}
-
-# Clones via SSH by default.
-ssh = True
-
-owner = Your Name
-
-[GITHUB]
-
-username = your-username
-
-# If some of the repositories hosted on GitHub are private, you can use
-# "personal access tokens" to access them.
-access_token = XXX
-
-[BITBUCKET]
-
-username = your-username
-
-# If some of the repositories hosted on Bitbucket are private, you can use "app
-# passwords" to access them.
-app_password = XXX
diff --git a/examples/cgitize.toml b/examples/cgitize.toml
new file mode 100644
index 0000000..ed89878
--- /dev/null
+++ b/examples/cgitize.toml
@@ -0,0 +1,46 @@
+# All settings are optional.
+
+# /var/tmp/cgitize/output by default.
+output = "/tmp/cgitize"
+
+# URL to clone from the output directory. {name} is replaced by the repository
+# ID.
+clone_url = "https://yourhost.com/git/{name}"
+
+# Clones via SSH by default.
+ssh = true
+
+# Unless this is specified, cgit is going to derive the owner from the
+# repository's directory ownership.
+owner = "Your Name"
+
+[github]
+# If some of the repositories hosted on GitHub are private, you can use
+# "personal access tokens" to access them.
+#access_token = "XXX"
+
+# Some random repositories hosted on GitHub:
+[github.repositories.lens]
+id = "ekmett/lens"
+[github.repositories.pytomlpp]
+id = "bobfang1992/pytomlpp"
+
+[bitbucket]
+# If some of the repositories hosted on Bitbucket are private, you can use "app
+# passwords" to access them.
+#app_password = "XXX"
+
+# Some random repositories hosted on Bitbucket:
+[bitbucket.repositories.cef]
+id = "chromiumembedded/cef"
+[bitbucket.repositories.upc-runtime]
+id = "berkeleylab/upc-runtime"
+
+[repositories]
+
+# Some random repositories hosted on the web:
+[repositories.wintun]
+id = "wintun"
+clone_url = "https://git.zx2c4.com/wintun"
+owner = "Jason A. Donenfeld"
+desc = "Layer 3 TUN Driver for Windows"
diff --git a/examples/my_repos.py b/examples/my_repos.py
deleted file mode 100644
index 1e83334..0000000
--- a/examples/my_repos.py
+++ /dev/null
@@ -1,12 +0,0 @@
-from cgitize.repo import Bitbucket, GitHub, Repo
-
-
-MY_REPOS = (
-    GitHub('xyz'),
-    GitHub('foo/bar', user='test', via_ssh=False),
-
-    Bitbucket('xyz'),
-    Bitbucket('foo/bar', desc='Foo (Bar)'),
-
-    Repo('tmp/tmp', clone_url='https://example.com/tmp.git', owner='John Doe'),
-)
diff --git a/requirements.txt b/requirements.txt
index f8d7d23..48a0a1f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,3 @@
 PyGithub ~= 1.0
 atlassian-python-api ~= 3.0
+tomli ~= 1.0
diff --git a/systemd/cgitize.service b/systemd/cgitize.service
index 0b517fc..c53ee7a 100644
--- a/systemd/cgitize.service
+++ b/systemd/cgitize.service
@@ -8,6 +8,6 @@ Wants=ssh-agent.service
 Type=simple
 WorkingDirectory=%h/workspace/personal/cgitize
 ExecStartPre=/usr/bin/truncate --size=0K -- %h/var/cgitize/cgitize.log
-ExecStart=/usr/bin/python3 -m cgitize.main --config %h/etc/cgitize/cgitize.conf
+ExecStart=/usr/bin/python3 -m cgitize.main --config %h/etc/cgitize/cgitize.toml
 StandardOutput=file:%h/var/cgitize/cgitize.log
 StandardError=inherit