aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authorEgor Tensin <Egor.Tensin@gmail.com>2021-07-31 10:16:38 +0300
committerEgor Tensin <Egor.Tensin@gmail.com>2021-07-31 10:17:44 +0300
commit9181b3a021acc7585cb6f9f20da6accfce0015e0 (patch)
tree0fcb4d5a99be02ac7a934817491f50d0d32d757d
parentadd some unit tests (diff)
downloadcgitize-9181b3a021acc7585cb6f9f20da6accfce0015e0.tar.gz
cgitize-9181b3a021acc7585cb6f9f20da6accfce0015e0.zip
merge my_repos.py to the config
The config is also in the TOML format now. It's a bit messy for the moment, but I'll fix it.
-rw-r--r--.ci/docker/client/etc/cgitize.toml4
-rw-r--r--.ci/docker/client/etc/my_repos.py6
-rwxr-xr-x.ci/local/test.sh43
-rw-r--r--.dockerignore1
-rw-r--r--Dockerfile7
-rw-r--r--README.md21
-rw-r--r--cgitize/cgit.py10
-rw-r--r--cgitize/config.py116
-rw-r--r--cgitize/main.py9
-rw-r--r--cgitize/repo.py76
-rw-r--r--examples/cgitize.conf34
-rw-r--r--examples/cgitize.toml46
-rw-r--r--examples/my_repos.py12
-rw-r--r--requirements.txt1
-rw-r--r--systemd/cgitize.service2
15 files changed, 198 insertions, 190 deletions
diff --git a/.ci/docker/client/etc/cgitize.toml b/.ci/docker/client/etc/cgitize.toml
new file mode 100644
index 0000000..bc46aca
--- /dev/null
+++ b/.ci/docker/client/etc/cgitize.toml
@@ -0,0 +1,4 @@
+[repositories.test_repo]
+
+id = "test_repo"
+clone_url = "root@server:~/test_repo"
diff --git a/.ci/docker/client/etc/my_repos.py b/.ci/docker/client/etc/my_repos.py
deleted file mode 100644
index 58cf542..0000000
--- a/.ci/docker/client/etc/my_repos.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from cgitize.repo import Repo
-
-
-MY_REPOS = (
- Repo('test_repo', clone_url='root@server:~/test_repo'),
-)
diff --git a/.ci/local/test.sh b/.ci/local/test.sh
index 8ae7fe9..8826ff4 100755
--- a/.ci/local/test.sh
+++ b/.ci/local/test.sh
@@ -8,8 +8,7 @@ readonly script_dir
upstream_repo_dir=
readonly etc_dir="$script_dir/etc"
-readonly cgitize_conf_path="$etc_dir/cgitize.conf"
-readonly my_repos_path="$etc_dir/my_repos.py"
+readonly cgitize_toml_path="$etc_dir/cgitize.toml"
readonly output_dir="$script_dir/output"
cleanup() {
@@ -61,47 +60,27 @@ add_commits() {
popd > /dev/null
}
-setup_cgitize_conf() {
+setup_cgitize_toml() {
echo
echo ----------------------------------------------------------------------
- echo cgitize.conf
+ echo cgitize.toml
echo ----------------------------------------------------------------------
local conf_dir
- conf_dir="$( dirname -- "$cgitize_conf_path" )"
+ conf_dir="$( dirname -- "$cgitize_toml_path" )"
mkdir -p -- "$conf_dir"
- cat <<EOF | tee "$cgitize_conf_path"
-[DEFAULT]
+ cat <<EOF | tee "$cgitize_toml_path"
+output = "$output_dir"
-my_repos = $( basename -- "$my_repos_path" )
-output = $output_dir
-EOF
-}
-
-setup_my_repos_py() {
- echo
- echo ----------------------------------------------------------------------
- echo my_repos.py
- echo ----------------------------------------------------------------------
-
- local conf_dir
- conf_dir="$( dirname -- "$my_repos_path" )"
- mkdir -p -- "$conf_dir"
-
- cat <<EOF | tee "$my_repos_path"
-from cgitize.repo import Repo
-
-
-MY_REPOS = (
- Repo('test_repo', clone_url='$upstream_repo_dir'),
-)
+[repositories.test_repo]
+id = "test_repo"
+clone_url = "$upstream_repo_dir"
EOF
}
setup_cgitize() {
- setup_cgitize_conf
- setup_my_repos_py
+ setup_cgitize_toml
}
setup_bare() {
@@ -127,7 +106,7 @@ cgitize() {
echo Running cgitize
echo ----------------------------------------------------------------------
- python3 -m cgitize.main --config "$cgitize_conf_path" --verbose
+ python3 -m cgitize.main --config "$cgitize_toml_path" --verbose
}
check_contains() {
diff --git a/.dockerignore b/.dockerignore
index eabf2d8..8f869af 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,3 +2,4 @@
!/cgitize/**
!/docker/**
+!/requirements.txt
diff --git a/Dockerfile b/Dockerfile
index b314d0f..5378dd1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,12 +1,17 @@
FROM alpine:3.13
-RUN apk add --no-cache bash git openssh-client python3 tini
+RUN build_deps='gcc libffi-dev make musl-dev python3-dev py3-pip' && \
+ runtime_deps='bash git openssh-client python3 tini' && \
+ apk add --no-cache $build_deps $runtime_deps
ARG ssh_sock_dir=/var/run/cgitize
ARG ssh_sock_path="$ssh_sock_dir/ssh-agent.sock"
ENV SSH_AUTH_SOCK "$ssh_sock_path"
+COPY ["requirements.txt", "/tmp/"]
+RUN pip3 install -r /tmp/requirements.txt
+
COPY ["docker/entrypoint.sh", "/"]
COPY ["cgitize/", "/usr/src/cgitize/"]
WORKDIR /usr/src
diff --git a/README.md b/README.md
index ee493b5..a74e4c0 100644
--- a/README.md
+++ b/README.md
@@ -15,28 +15,23 @@ Installation
Usage
-----
-cgitize uses two config files.
+Pass the path to the config to `cgitize` (/etc/cgitize/cgitize.toml by
+default):
-* cgitize.conf contains all the settings; see example at
-[examples/cgitize.conf].
-* my_repos.py contains the list of repositories to mirror; see example at
-[examples/my_repos.py].
+ cgitize --config path/to/cgitize.toml
-Pass the path to cgitize.conf using the `--config` parameter:
-
- cgitize --config path/to/cgitize.conf
+See an example config file at [examples/cgitize.toml].
cgitize uses the `git` executable, which might use `ssh` internally.
-Make sure the required keys are loaded to a ssh-agent (or use authentication
-tokens).
+Make sure the required keys are loaded to a ssh-agent (or use access
+tokens/application passwords).
-[examples/cgitize.conf]: examples/cgitize.conf
-[examples/my_repos.py]: examples/my_repos.py
+[examples/cgitize.toml]: examples/cgitize.toml
### Docker
The image is **egortensin/cgitize**.
-The container reads the config from /etc/cgitize/cgitize.conf and writes the
+The container reads the config from /etc/cgitize/cgitize.toml and writes the
repositories to /var/tmp/cgitize/output.
If SSH is required, the socket should be mapped to
/var/run/cgitize/ssh-agent.sock.
diff --git a/cgitize/cgit.py b/cgitize/cgit.py
index fe715d5..53a726a 100644
--- a/cgitize/cgit.py
+++ b/cgitize/cgit.py
@@ -18,7 +18,7 @@ class CGitServer:
def get_clone_url(self, repo):
if self.clone_url is None:
return None
- return self.clone_url.format(repo_id=repo.repo_id)
+ return self.clone_url.format(name=repo.name)
class CGitRCWriter:
@@ -96,7 +96,7 @@ class CGitRepositories:
return abs_path
def get_repo_dir(self, repo):
- return os.path.join(self.dir, repo.repo_id)
+ return os.path.join(self.dir, repo.name)
def update(self, repo):
success = self._mirror_or_update(repo)
@@ -130,7 +130,7 @@ class CGitRepositories:
# Jeez, there's a proper local repository in the target
# directory already with a different upstream; something's
# wrong, fix it manually.
- logging.warning("Existing repository '%s' doesn't match the specified clone URL: %s", repo.repo_id, repo.clone_url)
+ logging.warning("Existing repository '%s' doesn't match the specified clone URL: %s", repo.name, repo.clone_url)
if self.force:
# Unless --force was specified, in which case we overwrite
# the repository.
@@ -142,7 +142,7 @@ class CGitRepositories:
return self._update_existing(repo)
def _mirror(self, repo):
- logging.info("Mirroring repository '%s' from: %s", repo.repo_id, repo.clone_url)
+ logging.info("Mirroring repository '%s' from: %s", repo.name, repo.clone_url)
repo_dir = self.get_repo_dir(repo)
if os.path.isdir(repo_dir):
try:
@@ -159,7 +159,7 @@ class CGitRepositories:
return Git.check('remote', 'set-url', 'origin', repo.clone_url)
def _update_existing(self, repo):
- logging.info("Updating repository '%s'", repo.repo_id)
+ logging.info("Updating repository '%s'", repo.name)
repo_dir = self.get_repo_dir(repo)
with chdir(repo_dir):
with Git.setup_auth(repo):
diff --git a/cgitize/config.py b/cgitize/config.py
index a630d7a..8171860 100644
--- a/cgitize/config.py
+++ b/cgitize/config.py
@@ -9,73 +9,99 @@ import logging
import os.path
import sys
+from cgitize.repo import Repo, GitHub as GitHubRepo, Bitbucket as BitbucketRepo
from cgitize.utils import chdir
+import tomli
-class Config:
- DEFAULT_PATH = '/etc/cgitize/cgitize.conf'
- DEFAULT_OUTPUT_DIR = '/var/tmp/cgitize/output'
- DEFAULT_MY_REPOS_PATH = '/etc/cgitize/my_repos.py'
- @staticmethod
- def read(path):
- return Config(path)
+class Section:
+ def __init__(self, impl):
+ self.impl = impl
- def __init__(self, path):
- self.path = os.path.abspath(path)
- self.impl = configparser.ConfigParser()
- self.impl.read(path)
+ def _get_config_value(self, key, required=True, default=None):
+ if required and default is None:
+ if not key in self.impl:
+ raise RuntimeError(f'configuration value is missing: {key}')
+ return self.impl.get(key, default)
+
+ def _get_config_path(self, *args, **kwargs):
+ return os.path.abspath(self._get_config_value(*args, **kwargs))
- def _resolve_relative(self, path):
- if os.path.isabs(path):
- return path
- with chdir(os.path.dirname(self.path)):
- path = os.path.abspath(path)
- return path
+
+class Main(Section):
+ DEFAULT_OUTPUT_DIR = '/var/tmp/cgitize/output'
@property
def output(self):
- path = self.impl.get('DEFAULT', 'output', fallback=Config.DEFAULT_OUTPUT_DIR)
- return self._resolve_relative(path)
+ return self._get_config_path('output', default=Main.DEFAULT_OUTPUT_DIR)
@property
def clone_url(self):
- return self.impl.get('DEFAULT', 'clone_url', fallback=None)
+ return self._get_config_value('clone_url', required=False)
@property
def default_owner(self):
- return self.impl.get('DEFAULT', 'owner', fallback=None)
+ return self._get_config_value('owner', required=False)
@property
def via_ssh(self):
- return self.impl.getboolean('DEFAULT', 'ssh', fallback=True)
+ return self._get_config_value('ssh', default=True)
- @property
- def github_username(self):
- return self.impl.get('GITHUB', 'username', fallback=None)
- @property
- def github_access_token(self):
- return self.impl.get('GITHUB', 'access_token', fallback=None)
+class GitHub(Section):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.repositories = Repositories(self.impl.get('repositories', {}), GitHubRepo)
@property
- def bitbucket_username(self):
- return self.impl.get('BITBUCKET', 'username', fallback=None)
+ def access_token(self):
+ return self._get_config_value('access_token', required=False)
- @property
- def bitbucket_app_password(self):
- return self.impl.get('BITBUCKET', 'app_password', fallback=None)
+ def enum_repositories(self):
+ return self.repositories.enum_repositories()
+
+
+class Bitbucket(Section):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.repositories = Repositories(self.impl.get('repositories', {}), BitbucketRepo)
@property
- def my_repos(self):
- path = self.impl.get('DEFAULT', 'my_repos', fallback=Config.DEFAULT_MY_REPOS_PATH)
- return self._resolve_relative(path)
-
- def import_my_repos(self):
- sys.path.append(os.path.dirname(self.my_repos))
- if not os.path.exists(self.my_repos):
- logging.error("Couldn't find my_repos.py at: %s", self.my_repos)
- return None
- module_name = os.path.splitext(os.path.basename(self.my_repos))[0]
- module = importlib.import_module(module_name)
- return module.MY_REPOS
+ def app_password(self):
+ return self._get_config_value('app_password', required=False)
+
+ def enum_repositories(self):
+ return self.repositories.enum_repositories()
+
+
+class Repositories(Section):
+ def __init__(self, impl, repo_cls=Repo):
+ super().__init__(impl)
+ self.repo_cls = repo_cls
+
+ def enum_repositories(self):
+ for k, v in self.impl.items():
+ yield self.repo_cls.from_config(v)
+
+
+class Config:
+ DEFAULT_PATH = '/etc/cgitize/cgitize.toml'
+
+ @staticmethod
+ def read(path):
+ return Config(path)
+
+ def __init__(self, path):
+ self.path = os.path.abspath(path)
+ with open(self.path, 'rb') as f:
+ self.impl = tomli.load(f)
+ self.main = Main(self.impl)
+ self.repositories = Repositories(self.impl.get('repositories', {}))
+ self.github = GitHub(self.impl.get('github', {}))
+ self.bitbucket = Bitbucket(self.impl.get('bitbucket', {}))
+
+ def enum_repositories(self):
+ yield from self.repositories.enum_repositories()
+ yield from self.github.enum_repositories()
+ yield from self.bitbucket.enum_repositories()
diff --git a/cgitize/main.py b/cgitize/main.py
index 0c8f029..6b4705f 100644
--- a/cgitize/main.py
+++ b/cgitize/main.py
@@ -33,13 +33,10 @@ def main(args=None):
args = parse_args()
with setup_logging(args.verbose):
config = Config.read(args.config)
- my_repos = config.import_my_repos()
- if my_repos is None:
- return 1
- cgit_server = CGitServer(config.clone_url)
- output = CGitRepositories(config.output, cgit_server, force=args.force)
+ cgit_server = CGitServer(config.main.clone_url)
+ output = CGitRepositories(config.main.output, cgit_server, force=args.force)
success = True
- for repo in my_repos:
+ for repo in config.enum_repositories():
if args.repos is None or repo.repo_id in args.repos:
repo.fill_defaults(config)
repo.validate()
diff --git a/cgitize/repo.py b/cgitize/repo.py
index 8ecbcf5..01b7430 100644
--- a/cgitize/repo.py
+++ b/cgitize/repo.py
@@ -9,14 +9,17 @@ from urllib.parse import urlsplit, urlunsplit
class Repo:
- @staticmethod
- def extract_repo_name(repo_id):
- return os.path.basename(repo_id)
-
- def __init__(self, repo_id, clone_url=None, owner=None, desc=None,
+ @classmethod
+ def from_config(cls, cfg):
+ if 'id' not in cfg:
+ raise ValueError('every repository must have its id defined')
+ return cls(cfg['id'], clone_url=cfg.get('clone_url'),
+ owner=cfg.get('owner'), desc=cfg.get('desc'),
+ homepage=cfg.get('homepage'))
+
+ def __init__(self, name, clone_url=None, owner=None, desc=None,
homepage=None):
- self._repo_id = repo_id
- self._repo_name = self.extract_repo_name(repo_id)
+ self._name = name
self._clone_url = clone_url
self._owner = owner
self._desc = desc
@@ -24,19 +27,15 @@ class Repo:
def fill_defaults(self, config):
if self._owner is None:
- self._owner = config.default_owner
+ self._owner = config.main.default_owner
def validate(self):
if self.clone_url is None:
raise RuntimeError('upstream repository URL must be specified')
@property
- def repo_id(self):
- return self._repo_id
-
- @property
- def repo_name(self):
- return self._repo_name
+ def name(self):
+ return self._name
@property
def clone_url(self):
@@ -54,7 +53,7 @@ class Repo:
return self.homepage
if self.clone_url:
return self.clone_url
- return self.repo_name
+ return self.name
@property
def homepage(self):
@@ -66,21 +65,32 @@ class Repo:
class HostedRepo(Repo, abc.ABC):
+ @classmethod
+ def from_config(cls, cfg):
+ if 'id' not in cfg:
+ raise ValueError('every repository must have its id defined')
+ return cls(cfg['id'], owner=cfg.get('owner'), desc=cfg.get('desc'),
+ homepage=cfg.get('homepage'))
+
+ @staticmethod
+ def split_repo_id(repo_id):
+ components = repo_id.split('/')
+ if len(components) != 2:
+ raise ValueError(f'repository ID must be in the USER/NAME format: {repo_id}')
+ user, name = components
+ return user, name
+
def __init__(self, repo_id, owner=None, desc=None, homepage=None,
- user=None, via_ssh=True):
- super().__init__(repo_id, clone_url=None, owner=owner, desc=desc,
+ via_ssh=True):
+ user, name = self.split_repo_id(repo_id)
+ super().__init__(name, clone_url=None, owner=owner, desc=desc,
homepage=homepage)
self._user = user
self._via_ssh = via_ssh
def fill_defaults(self, config):
super().fill_defaults(config)
- self._via_ssh = config.via_ssh
-
- def validate(self):
- super().validate()
- if self.user is None:
- raise RuntimeError(f'neither explicit or default {self.provider_name} username was specified')
+ self._via_ssh = config.main.via_ssh
@property
def user(self):
@@ -127,9 +137,7 @@ class GitHub(HostedRepo):
def fill_defaults(self, config):
super().fill_defaults(config)
- if self._user is None:
- self._user = config.github_username
- self._access_token = config.github_access_token
+ self._access_token = config.github.access_token
@property
def provider_name(self):
@@ -137,7 +145,7 @@ class GitHub(HostedRepo):
@property
def homepage(self):
- return f'https://github.com/{self.user}/{self.repo_name}'
+ return f'https://github.com/{self.user}/{self.name}'
@property
def url_auth(self):
@@ -147,11 +155,11 @@ class GitHub(HostedRepo):
@property
def clone_url_ssh(self):
- return f'ssh://git@github.com/{self.user}/{self.repo_name}.git'
+ return f'ssh://git@github.com/{self.user}/{self.name}.git'
@property
def clone_url_https(self):
- return f'https://github.com/{self.user}/{self.repo_name}.git'
+ return f'https://github.com/{self.user}/{self.name}.git'
class Bitbucket(HostedRepo):
@@ -161,9 +169,7 @@ class Bitbucket(HostedRepo):
def fill_defaults(self, config):
super().fill_defaults(config)
- if self._user is None:
- self._user = config.bitbucket_username
- self._app_password = config.bitbucket_app_password
+ self._app_password = config.bitbucket.app_password
@property
def provider_name(self):
@@ -171,7 +177,7 @@ class Bitbucket(HostedRepo):
@property
def homepage(self):
- return f'https://bitbucket.org/{self.user}/{self.repo_name.lower()}'
+ return f'https://bitbucket.org/{self.user}/{self.name.lower()}'
@property
def url_auth(self):
@@ -181,8 +187,8 @@ class Bitbucket(HostedRepo):
@property
def clone_url_ssh(self):
- return f'ssh://git@bitbucket.org/{self.user}/{self.repo_name}.git'
+ return f'ssh://git@bitbucket.org/{self.user}/{self.name}.git'
@property
def clone_url_https(self):
- return f'https://bitbucket.org/{self.user}/{self.repo_name}.git'
+ return f'https://bitbucket.org/{self.user}/{self.name}.git'
diff --git a/examples/cgitize.conf b/examples/cgitize.conf
deleted file mode 100644
index a85daee..0000000
--- a/examples/cgitize.conf
+++ /dev/null
@@ -1,34 +0,0 @@
-# All settings are optional.
-
-[DEFAULT]
-
-# /etc/cgitize/my_repos.py by default.
-my_repos = /path/to/my_repos.py
-
-# /var/tmp/cgitize/output by default.
-output = /path/to/output/
-
-# URL to clone from the output directory. {repo_id} is replaced by the
-# repository ID (in the NAME or SECTION/NAME format).
-clone_url = http://example.com:8080/git/{repo_id}
-
-# Clones via SSH by default.
-ssh = True
-
-owner = Your Name
-
-[GITHUB]
-
-username = your-username
-
-# If some of the repositories hosted on GitHub are private, you can use
-# "personal access tokens" to access them.
-access_token = XXX
-
-[BITBUCKET]
-
-username = your-username
-
-# If some of the repositories hosted on Bitbucket are private, you can use "app
-# passwords" to access them.
-app_password = XXX
diff --git a/examples/cgitize.toml b/examples/cgitize.toml
new file mode 100644
index 0000000..ed89878
--- /dev/null
+++ b/examples/cgitize.toml
@@ -0,0 +1,46 @@
+# All settings are optional.
+
+# /var/tmp/cgitize/output by default.
+output = "/tmp/cgitize"
+
+# URL to clone from the output directory. {name} is replaced by the repository
+# ID.
+clone_url = "https://yourhost.com/git/{name}"
+
+# Clones via SSH by default.
+ssh = true
+
+# Unless this is specified, cgit is going to derive the owner from the
+# repository's directory ownership.
+owner = "Your Name"
+
+[github]
+# If some of the repositories hosted on GitHub are private, you can use
+# "personal access tokens" to access them.
+#access_token = "XXX"
+
+# Some random repositories hosted on GitHub:
+[github.repositories.lens]
+id = "ekmett/lens"
+[github.repositories.pytomlpp]
+id = "bobfang1992/pytomlpp"
+
+[bitbucket]
+# If some of the repositories hosted on Bitbucket are private, you can use "app
+# passwords" to access them.
+#app_password = "XXX"
+
+# Some random repositories hosted on Bitbucket:
+[bitbucket.repositories.cef]
+id = "chromiumembedded/cef"
+[bitbucket.repositories.upc-runtime]
+id = "berkeleylab/upc-runtime"
+
+[repositories]
+
+# Some random repositories hosted on the web:
+[repositories.wintun]
+id = "wintun"
+clone_url = "https://git.zx2c4.com/wintun"
+owner = "Jason A. Donenfeld"
+desc = "Layer 3 TUN Driver for Windows"
diff --git a/examples/my_repos.py b/examples/my_repos.py
deleted file mode 100644
index 1e83334..0000000
--- a/examples/my_repos.py
+++ /dev/null
@@ -1,12 +0,0 @@
-from cgitize.repo import Bitbucket, GitHub, Repo
-
-
-MY_REPOS = (
- GitHub('xyz'),
- GitHub('foo/bar', user='test', via_ssh=False),
-
- Bitbucket('xyz'),
- Bitbucket('foo/bar', desc='Foo (Bar)'),
-
- Repo('tmp/tmp', clone_url='https://example.com/tmp.git', owner='John Doe'),
-)
diff --git a/requirements.txt b/requirements.txt
index f8d7d23..48a0a1f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,3 @@
PyGithub ~= 1.0
atlassian-python-api ~= 3.0
+tomli ~= 1.0
diff --git a/systemd/cgitize.service b/systemd/cgitize.service
index 0b517fc..c53ee7a 100644
--- a/systemd/cgitize.service
+++ b/systemd/cgitize.service
@@ -8,6 +8,6 @@ Wants=ssh-agent.service
Type=simple
WorkingDirectory=%h/workspace/personal/cgitize
ExecStartPre=/usr/bin/truncate --size=0K -- %h/var/cgitize/cgitize.log
-ExecStart=/usr/bin/python3 -m cgitize.main --config %h/etc/cgitize/cgitize.conf
+ExecStart=/usr/bin/python3 -m cgitize.main --config %h/etc/cgitize/cgitize.toml
StandardOutput=file:%h/var/cgitize/cgitize.log
StandardError=inherit