aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRob Austein <sra@hactrn.net>2021-10-10 19:28:31 -0400
committerRob Austein <sra@hactrn.net>2021-10-10 19:28:31 -0400
commite51eac175bad30e1bfeaa172436195ef555e9884 (patch)
treeb34f3ee6ff67f5638bd953b749abe59599d8fe94
parent31ef459ab6a4ff627850bdda19564b2a566eca15 (diff)
Revise to use releng/alpha/.gitmodules rather than parsing TracHEADmaster
page that doesn't exist anymore.
-rwxr-xr-xhttps-sync-repos.py63
-rwxr-xr-xhttps-what-repos.py29
2 files changed, 50 insertions, 42 deletions
diff --git a/https-sync-repos.py b/https-sync-repos.py
index acd8694..0e78afe 100755
--- a/https-sync-repos.py
+++ b/https-sync-repos.py
@@ -1,38 +1,37 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
-# Synchronize Cryptech git repositories by scraping URLs from the
-# automatically generated Trac page. Yes, we know too much about how
-# the generation script and Trac format this page, c'est la vie.
+# Synchronize Cryptech git repositories by extracting URLs from the
+# Alpha release engineering repository's .gitmodules file. This
+# yields the same tree structure as used by releng/alpha, but without
+# the git submodule complexities.
-from urllib import urlopen
-from xml.etree.ElementTree import ElementTree
-from subprocess import check_call
-from os.path import isdir
-from sys import exit
+import sys, urllib.request, urllib.parse, subprocess
+from configparser import ConfigParser, DEFAULTSECT
+from pathlib import Path
-# URL for automatically generated Trac page listing repositories.
+errs = 0
-trac_page = "https://wiki.cryptech.is/wiki/GitRepositories"
+url = "https://git.cryptech.is/releng/alpha/plain/.gitmodules"
+cfg = ConfigParser(interpolation = None)
+cfg.read_string(urllib.request.urlopen(url).read().decode())
-head = "https://git.cryptech.is/"
-tail = ".git"
-errs = 0
+for section in cfg:
+ if section == DEFAULTSECT:
+ continue
+
+ url = cfg[section]["url"]
+ repo = Path(urllib.parse.urlparse(url).path).relative_to("/").with_suffix("")
+
+ try:
+ if repo.is_dir():
+ subprocess.run(("git", "-C", repo, "fetch", "--all", "--prune"), check = True)
+ subprocess.run(("git", "-C", repo, "merge", "--ff-only"), check = True)
+ else:
+ repo.parent.mkdir(parents = True, exist_ok = True)
+ subprocess.run(("git", "-C", repo.parent, "clone", url), check = True)
+
+ except:
+ print("** Error for repo", repo)
+ errs += 1
-for elt in ElementTree(file = urlopen(trac_page)).iter("{http://www.w3.org/1999/xhtml}code"):
- if elt.text.startswith(head) and elt.text.endswith(tail):
- url = elt.text
- repo = url[len(head):-len(tail)]
- pull = isdir(repo)
- print("")
- print(url)
- try:
- if pull:
- check_call(("git", "fetch", "--all", "--prune"), cwd = repo)
- check_call(("git", "merge", "--ff-only"), cwd = repo)
- else:
- check_call(("git", "clone", url, repo))
- except:
- print("** Error", "pulling" if pull else "cloning", repo)
- errs += 1
-
-exit(errs)
+sys.exit(errs)
diff --git a/https-what-repos.py b/https-what-repos.py
index 9237f76..f74d2f8 100755
--- a/https-what-repos.py
+++ b/https-what-repos.py
@@ -1,14 +1,23 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
-# List Cryptech git repository URLs by scraping them from the
-# automatically generated Trac page. Yes, we know too much about how
-# the generation script and Trac format this page, c'est la vie.
+# List Cryptech git repository URLs by extracting URLs from the
+# Alpha release engineering repository's .gitmodules file.
-from urllib import urlopen
-from xml.etree.ElementTree import ElementTree
+import urllib.request
+from configparser import ConfigParser, DEFAULTSECT
-url = "https://wiki.cryptech.is/wiki/GitRepositories"
+url = "https://git.cryptech.is/releng/alpha/plain/.gitmodules"
+cfg = ConfigParser(interpolation = None)
+cfg.read_string(urllib.request.urlopen(url).read().decode())
+
+urls = [
+ cfg[section]["url"]
+ for section in cfg
+ if section != DEFAULTSECT
+]
+
+urls.sort()
+
+for url in urls:
+ print(url)
-for x in ElementTree(file = urlopen(url)).iter("{http://www.w3.org/1999/xhtml}code"):
- if x.text.startswith("https://git.cryptech.is/") and x.text.endswith(".git"):
- print(x.text)