diff options
author | Egor Tensin <Egor.Tensin@gmail.com> | 2021-07-31 10:16:38 +0300 |
---|---|---|
committer | Egor Tensin <Egor.Tensin@gmail.com> | 2021-07-31 10:17:44 +0300 |
commit | 9181b3a021acc7585cb6f9f20da6accfce0015e0 (patch) | |
tree | 0fcb4d5a99be02ac7a934817491f50d0d32d757d | |
parent | add some unit tests (diff) | |
download | cgitize-9181b3a021acc7585cb6f9f20da6accfce0015e0.tar.gz cgitize-9181b3a021acc7585cb6f9f20da6accfce0015e0.zip |
merge my_repos.py to the config
The config is also in the TOML format now. It's a bit messy for the
moment, but I'll fix it.
-rw-r--r-- | .ci/docker/client/etc/cgitize.toml | 4 | ||||
-rw-r--r-- | .ci/docker/client/etc/my_repos.py | 6 | ||||
-rwxr-xr-x | .ci/local/test.sh | 43 | ||||
-rw-r--r-- | .dockerignore | 1 | ||||
-rw-r--r-- | Dockerfile | 7 | ||||
-rw-r--r-- | README.md | 21 | ||||
-rw-r--r-- | cgitize/cgit.py | 10 | ||||
-rw-r--r-- | cgitize/config.py | 116 | ||||
-rw-r--r-- | cgitize/main.py | 9 | ||||
-rw-r--r-- | cgitize/repo.py | 76 | ||||
-rw-r--r-- | examples/cgitize.conf | 34 | ||||
-rw-r--r-- | examples/cgitize.toml | 46 | ||||
-rw-r--r-- | examples/my_repos.py | 12 | ||||
-rw-r--r-- | requirements.txt | 1 | ||||
-rw-r--r-- | systemd/cgitize.service | 2 |
15 files changed, 198 insertions, 190 deletions
diff --git a/.ci/docker/client/etc/cgitize.toml b/.ci/docker/client/etc/cgitize.toml new file mode 100644 index 0000000..bc46aca --- /dev/null +++ b/.ci/docker/client/etc/cgitize.toml @@ -0,0 +1,4 @@ +[repositories.test_repo] + +id = "test_repo" +clone_url = "root@server:~/test_repo" diff --git a/.ci/docker/client/etc/my_repos.py b/.ci/docker/client/etc/my_repos.py deleted file mode 100644 index 58cf542..0000000 --- a/.ci/docker/client/etc/my_repos.py +++ /dev/null @@ -1,6 +0,0 @@ -from cgitize.repo import Repo - - -MY_REPOS = ( - Repo('test_repo', clone_url='root@server:~/test_repo'), -) diff --git a/.ci/local/test.sh b/.ci/local/test.sh index 8ae7fe9..8826ff4 100755 --- a/.ci/local/test.sh +++ b/.ci/local/test.sh @@ -8,8 +8,7 @@ readonly script_dir upstream_repo_dir= readonly etc_dir="$script_dir/etc" -readonly cgitize_conf_path="$etc_dir/cgitize.conf" -readonly my_repos_path="$etc_dir/my_repos.py" +readonly cgitize_toml_path="$etc_dir/cgitize.toml" readonly output_dir="$script_dir/output" cleanup() { @@ -61,47 +60,27 @@ add_commits() { popd > /dev/null } -setup_cgitize_conf() { +setup_cgitize_toml() { echo echo ---------------------------------------------------------------------- - echo cgitize.conf + echo cgitize.toml echo ---------------------------------------------------------------------- local conf_dir - conf_dir="$( dirname -- "$cgitize_conf_path" )" + conf_dir="$( dirname -- "$cgitize_toml_path" )" mkdir -p -- "$conf_dir" - cat <<EOF | tee "$cgitize_conf_path" -[DEFAULT] + cat <<EOF | tee "$cgitize_toml_path" +output = "$output_dir" -my_repos = $( basename -- "$my_repos_path" ) -output = $output_dir -EOF -} - -setup_my_repos_py() { - echo - echo ---------------------------------------------------------------------- - echo my_repos.py - echo ---------------------------------------------------------------------- - - local conf_dir - conf_dir="$( dirname -- "$my_repos_path" )" - mkdir -p -- "$conf_dir" - - cat <<EOF | tee "$my_repos_path" -from cgitize.repo import Repo - - -MY_REPOS = ( - Repo('test_repo', clone_url='$upstream_repo_dir'), -) +[repositories.test_repo] +id = "test_repo" +clone_url = "$upstream_repo_dir" EOF } setup_cgitize() { - setup_cgitize_conf - setup_my_repos_py + setup_cgitize_toml } setup_bare() { @@ -127,7 +106,7 @@ cgitize() { echo Running cgitize echo ---------------------------------------------------------------------- - python3 -m cgitize.main --config "$cgitize_conf_path" --verbose + python3 -m cgitize.main --config "$cgitize_toml_path" --verbose } check_contains() { diff --git a/.dockerignore b/.dockerignore index eabf2d8..8f869af 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,4 @@ !/cgitize/** !/docker/** +!/requirements.txt @@ -1,12 +1,17 @@ FROM alpine:3.13 -RUN apk add --no-cache bash git openssh-client python3 tini +RUN build_deps='gcc libffi-dev make musl-dev python3-dev py3-pip' && \ + runtime_deps='bash git openssh-client python3 tini' && \ + apk add --no-cache $build_deps $runtime_deps ARG ssh_sock_dir=/var/run/cgitize ARG ssh_sock_path="$ssh_sock_dir/ssh-agent.sock" ENV SSH_AUTH_SOCK "$ssh_sock_path" +COPY ["requirements.txt", "/tmp/"] +RUN pip3 install -r /tmp/requirements.txt + COPY ["docker/entrypoint.sh", "/"] COPY ["cgitize/", "/usr/src/cgitize/"] WORKDIR /usr/src @@ -15,28 +15,23 @@ Installation Usage ----- -cgitize uses two config files. +Pass the path to the config to `cgitize` (/etc/cgitize/cgitize.toml by +default): -* cgitize.conf contains all the settings; see example at -[examples/cgitize.conf]. -* my_repos.py contains the list of repositories to mirror; see example at -[examples/my_repos.py]. + cgitize --config path/to/cgitize.toml -Pass the path to cgitize.conf using the `--config` parameter: - - cgitize --config path/to/cgitize.conf +See an example config file at [examples/cgitize.toml]. cgitize uses the `git` executable, which might use `ssh` internally. -Make sure the required keys are loaded to a ssh-agent (or use authentication -tokens). +Make sure the required keys are loaded to a ssh-agent (or use access +tokens/application passwords). -[examples/cgitize.conf]: examples/cgitize.conf -[examples/my_repos.py]: examples/my_repos.py +[examples/cgitize.toml]: examples/cgitize.toml ### Docker The image is **egortensin/cgitize**. -The container reads the config from /etc/cgitize/cgitize.conf and writes the +The container reads the config from /etc/cgitize/cgitize.toml and writes the repositories to /var/tmp/cgitize/output. If SSH is required, the socket should be mapped to /var/run/cgitize/ssh-agent.sock. diff --git a/cgitize/cgit.py b/cgitize/cgit.py index fe715d5..53a726a 100644 --- a/cgitize/cgit.py +++ b/cgitize/cgit.py @@ -18,7 +18,7 @@ class CGitServer: def get_clone_url(self, repo): if self.clone_url is None: return None - return self.clone_url.format(repo_id=repo.repo_id) + return self.clone_url.format(name=repo.name) class CGitRCWriter: @@ -96,7 +96,7 @@ class CGitRepositories: return abs_path def get_repo_dir(self, repo): - return os.path.join(self.dir, repo.repo_id) + return os.path.join(self.dir, repo.name) def update(self, repo): success = self._mirror_or_update(repo) @@ -130,7 +130,7 @@ class CGitRepositories: # Jeez, there's a proper local repository in the target # directory already with a different upstream; something's # wrong, fix it manually. - logging.warning("Existing repository '%s' doesn't match the specified clone URL: %s", repo.repo_id, repo.clone_url) + logging.warning("Existing repository '%s' doesn't match the specified clone URL: %s", repo.name, repo.clone_url) if self.force: # Unless --force was specified, in which case we overwrite # the repository. @@ -142,7 +142,7 @@ class CGitRepositories: return self._update_existing(repo) def _mirror(self, repo): - logging.info("Mirroring repository '%s' from: %s", repo.repo_id, repo.clone_url) + logging.info("Mirroring repository '%s' from: %s", repo.name, repo.clone_url) repo_dir = self.get_repo_dir(repo) if os.path.isdir(repo_dir): try: @@ -159,7 +159,7 @@ class CGitRepositories: return Git.check('remote', 'set-url', 'origin', repo.clone_url) def _update_existing(self, repo): - logging.info("Updating repository '%s'", repo.repo_id) + logging.info("Updating repository '%s'", repo.name) repo_dir = self.get_repo_dir(repo) with chdir(repo_dir): with Git.setup_auth(repo): diff --git a/cgitize/config.py b/cgitize/config.py index a630d7a..8171860 100644 --- a/cgitize/config.py +++ b/cgitize/config.py @@ -9,73 +9,99 @@ import logging import os.path import sys +from cgitize.repo import Repo, GitHub as GitHubRepo, Bitbucket as BitbucketRepo from cgitize.utils import chdir +import tomli -class Config: - DEFAULT_PATH = '/etc/cgitize/cgitize.conf' - DEFAULT_OUTPUT_DIR = '/var/tmp/cgitize/output' - DEFAULT_MY_REPOS_PATH = '/etc/cgitize/my_repos.py' - @staticmethod - def read(path): - return Config(path) +class Section: + def __init__(self, impl): + self.impl = impl - def __init__(self, path): - self.path = os.path.abspath(path) - self.impl = configparser.ConfigParser() - self.impl.read(path) + def _get_config_value(self, key, required=True, default=None): + if required and default is None: + if not key in self.impl: + raise RuntimeError(f'configuration value is missing: {key}') + return self.impl.get(key, default) + + def _get_config_path(self, *args, **kwargs): + return os.path.abspath(self._get_config_value(*args, **kwargs)) - def _resolve_relative(self, path): - if os.path.isabs(path): - return path - with chdir(os.path.dirname(self.path)): - path = os.path.abspath(path) - return path + +class Main(Section): + DEFAULT_OUTPUT_DIR = '/var/tmp/cgitize/output' @property def output(self): - path = self.impl.get('DEFAULT', 'output', fallback=Config.DEFAULT_OUTPUT_DIR) - return self._resolve_relative(path) + return self._get_config_path('output', default=Main.DEFAULT_OUTPUT_DIR) @property def clone_url(self): - return self.impl.get('DEFAULT', 'clone_url', fallback=None) + return self._get_config_value('clone_url', required=False) @property def default_owner(self): - return self.impl.get('DEFAULT', 'owner', fallback=None) + return self._get_config_value('owner', required=False) @property def via_ssh(self): - return self.impl.getboolean('DEFAULT', 'ssh', fallback=True) + return self._get_config_value('ssh', default=True) - @property - def github_username(self): - return self.impl.get('GITHUB', 'username', fallback=None) - @property - def github_access_token(self): - return self.impl.get('GITHUB', 'access_token', fallback=None) +class GitHub(Section): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.repositories = Repositories(self.impl.get('repositories', {}), GitHubRepo) @property - def bitbucket_username(self): - return self.impl.get('BITBUCKET', 'username', fallback=None) + def access_token(self): + return self._get_config_value('access_token', required=False) - @property - def bitbucket_app_password(self): - return self.impl.get('BITBUCKET', 'app_password', fallback=None) + def enum_repositories(self): + return self.repositories.enum_repositories() + + +class Bitbucket(Section): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.repositories = Repositories(self.impl.get('repositories', {}), BitbucketRepo) @property - def my_repos(self): - path = self.impl.get('DEFAULT', 'my_repos', fallback=Config.DEFAULT_MY_REPOS_PATH) - return self._resolve_relative(path) - - def import_my_repos(self): - sys.path.append(os.path.dirname(self.my_repos)) - if not os.path.exists(self.my_repos): - logging.error("Couldn't find my_repos.py at: %s", self.my_repos) - return None - module_name = os.path.splitext(os.path.basename(self.my_repos))[0] - module = importlib.import_module(module_name) - return module.MY_REPOS + def app_password(self): + return self._get_config_value('app_password', required=False) + + def enum_repositories(self): + return self.repositories.enum_repositories() + + +class Repositories(Section): + def __init__(self, impl, repo_cls=Repo): + super().__init__(impl) + self.repo_cls = repo_cls + + def enum_repositories(self): + for k, v in self.impl.items(): + yield self.repo_cls.from_config(v) + + +class Config: + DEFAULT_PATH = '/etc/cgitize/cgitize.toml' + + @staticmethod + def read(path): + return Config(path) + + def __init__(self, path): + self.path = os.path.abspath(path) + with open(self.path, 'rb') as f: + self.impl = tomli.load(f) + self.main = Main(self.impl) + self.repositories = Repositories(self.impl.get('repositories', {})) + self.github = GitHub(self.impl.get('github', {})) + self.bitbucket = Bitbucket(self.impl.get('bitbucket', {})) + + def enum_repositories(self): + yield from self.repositories.enum_repositories() + yield from self.github.enum_repositories() + yield from self.bitbucket.enum_repositories() diff --git a/cgitize/main.py b/cgitize/main.py index 0c8f029..6b4705f 100644 --- a/cgitize/main.py +++ b/cgitize/main.py @@ -33,13 +33,10 @@ def main(args=None): args = parse_args() with setup_logging(args.verbose): config = Config.read(args.config) - my_repos = config.import_my_repos() - if my_repos is None: - return 1 - cgit_server = CGitServer(config.clone_url) - output = CGitRepositories(config.output, cgit_server, force=args.force) + cgit_server = CGitServer(config.main.clone_url) + output = CGitRepositories(config.main.output, cgit_server, force=args.force) success = True - for repo in my_repos: + for repo in config.enum_repositories(): if args.repos is None or repo.repo_id in args.repos: repo.fill_defaults(config) repo.validate() diff --git a/cgitize/repo.py b/cgitize/repo.py index 8ecbcf5..01b7430 100644 --- a/cgitize/repo.py +++ b/cgitize/repo.py @@ -9,14 +9,17 @@ from urllib.parse import urlsplit, urlunsplit class Repo: - @staticmethod - def extract_repo_name(repo_id): - return os.path.basename(repo_id) - - def __init__(self, repo_id, clone_url=None, owner=None, desc=None, + @classmethod + def from_config(cls, cfg): + if 'id' not in cfg: + raise ValueError('every repository must have its id defined') + return cls(cfg['id'], clone_url=cfg.get('clone_url'), + owner=cfg.get('owner'), desc=cfg.get('desc'), + homepage=cfg.get('homepage')) + + def __init__(self, name, clone_url=None, owner=None, desc=None, homepage=None): - self._repo_id = repo_id - self._repo_name = self.extract_repo_name(repo_id) + self._name = name self._clone_url = clone_url self._owner = owner self._desc = desc @@ -24,19 +27,15 @@ class Repo: def fill_defaults(self, config): if self._owner is None: - self._owner = config.default_owner + self._owner = config.main.default_owner def validate(self): if self.clone_url is None: raise RuntimeError('upstream repository URL must be specified') @property - def repo_id(self): - return self._repo_id - - @property - def repo_name(self): - return self._repo_name + def name(self): + return self._name @property def clone_url(self): @@ -54,7 +53,7 @@ class Repo: return self.homepage if self.clone_url: return self.clone_url - return self.repo_name + return self.name @property def homepage(self): @@ -66,21 +65,32 @@ class Repo: class HostedRepo(Repo, abc.ABC): + @classmethod + def from_config(cls, cfg): + if 'id' not in cfg: + raise ValueError('every repository must have its id defined') + return cls(cfg['id'], owner=cfg.get('owner'), desc=cfg.get('desc'), + homepage=cfg.get('homepage')) + + @staticmethod + def split_repo_id(repo_id): + components = repo_id.split('/') + if len(components) != 2: + raise ValueError(f'repository ID must be in the USER/NAME format: {repo_id}') + user, name = components + return user, name + def __init__(self, repo_id, owner=None, desc=None, homepage=None, - user=None, via_ssh=True): - super().__init__(repo_id, clone_url=None, owner=owner, desc=desc, + via_ssh=True): + user, name = self.split_repo_id(repo_id) + super().__init__(name, clone_url=None, owner=owner, desc=desc, homepage=homepage) self._user = user self._via_ssh = via_ssh def fill_defaults(self, config): super().fill_defaults(config) - self._via_ssh = config.via_ssh - - def validate(self): - super().validate() - if self.user is None: - raise RuntimeError(f'neither explicit or default {self.provider_name} username was specified') + self._via_ssh = config.main.via_ssh @property def user(self): @@ -127,9 +137,7 @@ class GitHub(HostedRepo): def fill_defaults(self, config): super().fill_defaults(config) - if self._user is None: - self._user = config.github_username - self._access_token = config.github_access_token + self._access_token = config.github.access_token @property def provider_name(self): @@ -137,7 +145,7 @@ class GitHub(HostedRepo): @property def homepage(self): - return f'https://github.com/{self.user}/{self.repo_name}' + return f'https://github.com/{self.user}/{self.name}' @property def url_auth(self): @@ -147,11 +155,11 @@ class GitHub(HostedRepo): @property def clone_url_ssh(self): - return f'ssh://git@github.com/{self.user}/{self.repo_name}.git' + return f'ssh://git@github.com/{self.user}/{self.name}.git' @property def clone_url_https(self): - return f'https://github.com/{self.user}/{self.repo_name}.git' + return f'https://github.com/{self.user}/{self.name}.git' class Bitbucket(HostedRepo): @@ -161,9 +169,7 @@ class Bitbucket(HostedRepo): def fill_defaults(self, config): super().fill_defaults(config) - if self._user is None: - self._user = config.bitbucket_username - self._app_password = config.bitbucket_app_password + self._app_password = config.bitbucket.app_password @property def provider_name(self): @@ -171,7 +177,7 @@ class Bitbucket(HostedRepo): @property def homepage(self): - return f'https://bitbucket.org/{self.user}/{self.repo_name.lower()}' + return f'https://bitbucket.org/{self.user}/{self.name.lower()}' @property def url_auth(self): @@ -181,8 +187,8 @@ class Bitbucket(HostedRepo): @property def clone_url_ssh(self): - return f'ssh://git@bitbucket.org/{self.user}/{self.repo_name}.git' + return f'ssh://git@bitbucket.org/{self.user}/{self.name}.git' @property def clone_url_https(self): - return f'https://bitbucket.org/{self.user}/{self.repo_name}.git' + return f'https://bitbucket.org/{self.user}/{self.name}.git' diff --git a/examples/cgitize.conf b/examples/cgitize.conf deleted file mode 100644 index a85daee..0000000 --- a/examples/cgitize.conf +++ /dev/null @@ -1,34 +0,0 @@ -# All settings are optional. - -[DEFAULT] - -# /etc/cgitize/my_repos.py by default. -my_repos = /path/to/my_repos.py - -# /var/tmp/cgitize/output by default. -output = /path/to/output/ - -# URL to clone from the output directory. {repo_id} is replaced by the -# repository ID (in the NAME or SECTION/NAME format). -clone_url = http://example.com:8080/git/{repo_id} - -# Clones via SSH by default. -ssh = True - -owner = Your Name - -[GITHUB] - -username = your-username - -# If some of the repositories hosted on GitHub are private, you can use -# "personal access tokens" to access them. -access_token = XXX - -[BITBUCKET] - -username = your-username - -# If some of the repositories hosted on Bitbucket are private, you can use "app -# passwords" to access them. -app_password = XXX diff --git a/examples/cgitize.toml b/examples/cgitize.toml new file mode 100644 index 0000000..ed89878 --- /dev/null +++ b/examples/cgitize.toml @@ -0,0 +1,46 @@ +# All settings are optional. + +# /var/tmp/cgitize/output by default. +output = "/tmp/cgitize" + +# URL to clone from the output directory. {name} is replaced by the repository +# ID. +clone_url = "https://yourhost.com/git/{name}" + +# Clones via SSH by default. +ssh = true + +# Unless this is specified, cgit is going to derive the owner from the +# repository's directory ownership. +owner = "Your Name" + +[github] +# If some of the repositories hosted on GitHub are private, you can use +# "personal access tokens" to access them. +#access_token = "XXX" + +# Some random repositories hosted on GitHub: +[github.repositories.lens] +id = "ekmett/lens" +[github.repositories.pytomlpp] +id = "bobfang1992/pytomlpp" + +[bitbucket] +# If some of the repositories hosted on Bitbucket are private, you can use "app +# passwords" to access them. +#app_password = "XXX" + +# Some random repositories hosted on Bitbucket: +[bitbucket.repositories.cef] +id = "chromiumembedded/cef" +[bitbucket.repositories.upc-runtime] +id = "berkeleylab/upc-runtime" + +[repositories] + +# Some random repositories hosted on the web: +[repositories.wintun] +id = "wintun" +clone_url = "https://git.zx2c4.com/wintun" +owner = "Jason A. Donenfeld" +desc = "Layer 3 TUN Driver for Windows" diff --git a/examples/my_repos.py b/examples/my_repos.py deleted file mode 100644 index 1e83334..0000000 --- a/examples/my_repos.py +++ /dev/null @@ -1,12 +0,0 @@ -from cgitize.repo import Bitbucket, GitHub, Repo - - -MY_REPOS = ( - GitHub('xyz'), - GitHub('foo/bar', user='test', via_ssh=False), - - Bitbucket('xyz'), - Bitbucket('foo/bar', desc='Foo (Bar)'), - - Repo('tmp/tmp', clone_url='https://example.com/tmp.git', owner='John Doe'), -) diff --git a/requirements.txt b/requirements.txt index f8d7d23..48a0a1f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ PyGithub ~= 1.0 atlassian-python-api ~= 3.0 +tomli ~= 1.0 diff --git a/systemd/cgitize.service b/systemd/cgitize.service index 0b517fc..c53ee7a 100644 --- a/systemd/cgitize.service +++ b/systemd/cgitize.service @@ -8,6 +8,6 @@ Wants=ssh-agent.service Type=simple WorkingDirectory=%h/workspace/personal/cgitize ExecStartPre=/usr/bin/truncate --size=0K -- %h/var/cgitize/cgitize.log -ExecStart=/usr/bin/python3 -m cgitize.main --config %h/etc/cgitize/cgitize.conf +ExecStart=/usr/bin/python3 -m cgitize.main --config %h/etc/cgitize/cgitize.toml StandardOutput=file:%h/var/cgitize/cgitize.log StandardError=inherit |