From e51eac175bad30e1bfeaa172436195ef555e9884 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Sun, 10 Oct 2021 19:28:31 -0400 Subject: Revise to use releng/alpha/.gitmodules rather than parsing Trac page that doesn't exist anymore. --- https-sync-repos.py | 63 ++++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 32 deletions(-) (limited to 'https-sync-repos.py') diff --git a/https-sync-repos.py b/https-sync-repos.py index acd8694..0e78afe 100755 --- a/https-sync-repos.py +++ b/https-sync-repos.py @@ -1,38 +1,37 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Synchronize Cryptech git repositories by scraping URLs from the -# automatically generated Trac page. Yes, we know too much about how -# the generation script and Trac format this page, c'est la vie. +# Synchronize Cryptech git repositories by extracting URLs from the +# Alpha release engineering repository's .gitmodules file. This +# yields the same tree structure as used by releng/alpha, but without +# the git submodule complexities. -from urllib import urlopen -from xml.etree.ElementTree import ElementTree -from subprocess import check_call -from os.path import isdir -from sys import exit +import sys, urllib.request, urllib.parse, subprocess +from configparser import ConfigParser, DEFAULTSECT +from pathlib import Path -# URL for automatically generated Trac page listing repositories. +errs = 0 -trac_page = "https://wiki.cryptech.is/wiki/GitRepositories" +url = "https://git.cryptech.is/releng/alpha/plain/.gitmodules" +cfg = ConfigParser(interpolation = None) +cfg.read_string(urllib.request.urlopen(url).read().decode()) -head = "https://git.cryptech.is/" -tail = ".git" -errs = 0 +for section in cfg: + if section == DEFAULTSECT: + continue + + url = cfg[section]["url"] + repo = Path(urllib.parse.urlparse(url).path).relative_to("/").with_suffix("") + + try: + if repo.is_dir(): + subprocess.run(("git", "-C", repo, "fetch", "--all", "--prune"), check = True) + subprocess.run(("git", "-C", repo, "merge", "--ff-only"), check = True) + else: + repo.parent.mkdir(parents = True, exist_ok = True) + subprocess.run(("git", "-C", repo.parent, "clone", url), check = True) + + except: + print("** Error for repo", repo) + errs += 1 -for elt in ElementTree(file = urlopen(trac_page)).iter("{http://www.w3.org/1999/xhtml}code"): - if elt.text.startswith(head) and elt.text.endswith(tail): - url = elt.text - repo = url[len(head):-len(tail)] - pull = isdir(repo) - print("") - print(url) - try: - if pull: - check_call(("git", "fetch", "--all", "--prune"), cwd = repo) - check_call(("git", "merge", "--ff-only"), cwd = repo) - else: - check_call(("git", "clone", url, repo)) - except: - print("** Error", "pulling" if pull else "cloning", repo) - errs += 1 - -exit(errs) +sys.exit(errs) -- cgit v1.2.3