Revise to use releng/alpha/.gitmodules rather than parsing TracHEADmaster

page that doesn't exist anymore.

2 files changed, 50 insertions, 42 deletions

diff --git a/https-sync-repos.py b/https-sync-repos.py
index acd8694..0e78afe 100755
--- a/https-sync-repos.py
+++ b/https-sync-repos.py
@@ -1,38 +1,37 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
-# Synchronize Cryptech git repositories by scraping URLs from the
-# automatically generated Trac page.  Yes, we know too much about how
-# the generation script and Trac format this page, c'est la vie.
+# Synchronize Cryptech git repositories by extracting URLs from the
+# Alpha release engineering repository's .gitmodules file.  This
+# yields the same tree structure as used by releng/alpha, but without
+# the git submodule complexities.
 
-from urllib                     import urlopen
-from xml.etree.ElementTree      import ElementTree
-from subprocess                 import check_call
-from os.path                    import isdir
-from sys                        import exit
+import sys, urllib.request, urllib.parse, subprocess
+from configparser import ConfigParser, DEFAULTSECT
+from pathlib import Path
 
-# URL for automatically generated Trac page listing repositories.
+errs = 0
 
-trac_page = "https://wiki.cryptech.is/wiki/GitRepositories"
+url = "https://git.cryptech.is/releng/alpha/plain/.gitmodules"
+cfg = ConfigParser(interpolation = None)
+cfg.read_string(urllib.request.urlopen(url).read().decode())
 
-head = "https://git.cryptech.is/"
-tail = ".git"
-errs = 0
+for section in cfg:
+    if section == DEFAULTSECT:
+        continue
+
+    url = cfg[section]["url"]
+    repo = Path(urllib.parse.urlparse(url).path).relative_to("/").with_suffix("")
+
+    try:
+        if repo.is_dir():
+            subprocess.run(("git", "-C", repo, "fetch", "--all", "--prune"), check = True)
+            subprocess.run(("git", "-C", repo, "merge", "--ff-only"), check = True)
+        else:
+            repo.parent.mkdir(parents = True, exist_ok = True)
+            subprocess.run(("git", "-C", repo.parent, "clone", url), check = True)
+
+    except:
+        print("** Error for repo", repo)
+        errs += 1
 
-for elt in ElementTree(file = urlopen(trac_page)).iter("{http://www.w3.org/1999/xhtml}code"):
-    if elt.text.startswith(head) and elt.text.endswith(tail):
-        url  = elt.text
-        repo = url[len(head):-len(tail)]
-        pull = isdir(repo)
-        print("")
-        print(url)
-        try:
-            if pull:
-                check_call(("git", "fetch", "--all", "--prune"), cwd = repo)
-                check_call(("git", "merge", "--ff-only"), cwd = repo)
-            else:
-                check_call(("git", "clone", url, repo))
-        except:
-            print("** Error", "pulling" if pull else "cloning", repo)
-            errs += 1
-
-exit(errs)
+sys.exit(errs)
diff --git a/https-what-repos.py b/https-what-repos.py
index 9237f76..f74d2f8 100755
--- a/https-what-repos.py
+++ b/https-what-repos.py
@@ -1,14 +1,23 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
-# List Cryptech git repository URLs by scraping them from the
-# automatically generated Trac page.  Yes, we know too much about how
-# the generation script and Trac format this page, c'est la vie.
+# List Cryptech git repository URLs by extracting URLs from the
+# Alpha release engineering repository's .gitmodules file.
 
-from urllib                     import urlopen
-from xml.etree.ElementTree      import ElementTree
+import urllib.request
+from configparser import ConfigParser, DEFAULTSECT
 
-url = "https://wiki.cryptech.is/wiki/GitRepositories"
+url = "https://git.cryptech.is/releng/alpha/plain/.gitmodules"
+cfg = ConfigParser(interpolation = None)
+cfg.read_string(urllib.request.urlopen(url).read().decode())
+
+urls = [
+    cfg[section]["url"]
+    for section in cfg
+    if section != DEFAULTSECT
+]
+
+urls.sort()
+
+for url in urls:
+    print(url)
 
-for x in ElementTree(file = urlopen(url)).iter("{http://www.w3.org/1999/xhtml}code"):
-    if x.text.startswith("https://git.cryptech.is/") and x.text.endswith(".git"):
-        print(x.text)