diff options
| -rwxr-xr-x | https-sync-repos.py | 63 | ||||
| -rwxr-xr-x | https-what-repos.py | 29 |
2 files changed, 50 insertions, 42 deletions
diff --git a/https-sync-repos.py b/https-sync-repos.py index acd8694..0e78afe 100755 --- a/https-sync-repos.py +++ b/https-sync-repos.py @@ -1,38 +1,37 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Synchronize Cryptech git repositories by scraping URLs from the -# automatically generated Trac page. Yes, we know too much about how -# the generation script and Trac format this page, c'est la vie. +# Synchronize Cryptech git repositories by extracting URLs from the +# Alpha release engineering repository's .gitmodules file. This +# yields the same tree structure as used by releng/alpha, but without +# the git submodule complexities. -from urllib import urlopen -from xml.etree.ElementTree import ElementTree -from subprocess import check_call -from os.path import isdir -from sys import exit +import sys, urllib.request, urllib.parse, subprocess +from configparser import ConfigParser, DEFAULTSECT +from pathlib import Path -# URL for automatically generated Trac page listing repositories. +errs = 0 -trac_page = "https://wiki.cryptech.is/wiki/GitRepositories" +url = "https://git.cryptech.is/releng/alpha/plain/.gitmodules" +cfg = ConfigParser(interpolation = None) +cfg.read_string(urllib.request.urlopen(url).read().decode()) -head = "https://git.cryptech.is/" -tail = ".git" -errs = 0 +for section in cfg: + if section == DEFAULTSECT: + continue + + url = cfg[section]["url"] + repo = Path(urllib.parse.urlparse(url).path).relative_to("/").with_suffix("") + + try: + if repo.is_dir(): + subprocess.run(("git", "-C", repo, "fetch", "--all", "--prune"), check = True) + subprocess.run(("git", "-C", repo, "merge", "--ff-only"), check = True) + else: + repo.parent.mkdir(parents = True, exist_ok = True) + subprocess.run(("git", "-C", repo.parent, "clone", url), check = True) + + except: + print("** Error for repo", repo) + errs += 1 -for elt in ElementTree(file = urlopen(trac_page)).iter("{http://www.w3.org/1999/xhtml}code"): - if elt.text.startswith(head) and elt.text.endswith(tail): - url = elt.text - repo = url[len(head):-len(tail)] - pull = isdir(repo) - print("") - print(url) - try: - if pull: - check_call(("git", "fetch", "--all", "--prune"), cwd = repo) - check_call(("git", "merge", "--ff-only"), cwd = repo) - else: - check_call(("git", "clone", url, repo)) - except: - print("** Error", "pulling" if pull else "cloning", repo) - errs += 1 - -exit(errs) +sys.exit(errs) diff --git a/https-what-repos.py b/https-what-repos.py index 9237f76..f74d2f8 100755 --- a/https-what-repos.py +++ b/https-what-repos.py @@ -1,14 +1,23 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# List Cryptech git repository URLs by scraping them from the -# automatically generated Trac page. Yes, we know too much about how -# the generation script and Trac format this page, c'est la vie. +# List Cryptech git repository URLs by extracting URLs from the +# Alpha release engineering repository's .gitmodules file. -from urllib import urlopen -from xml.etree.ElementTree import ElementTree +import urllib.request +from configparser import ConfigParser, DEFAULTSECT -url = "https://wiki.cryptech.is/wiki/GitRepositories" +url = "https://git.cryptech.is/releng/alpha/plain/.gitmodules" +cfg = ConfigParser(interpolation = None) +cfg.read_string(urllib.request.urlopen(url).read().decode()) + +urls = [ + cfg[section]["url"] + for section in cfg + if section != DEFAULTSECT +] + +urls.sort() + +for url in urls: + print(url) -for x in ElementTree(file = urlopen(url)).iter("{http://www.w3.org/1999/xhtml}code"): - if x.text.startswith("https://git.cryptech.is/") and x.text.endswith(".git"): - print(x.text) |