From ef27539ad71977b06683bbc304760f366ed4199e Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Sat, 18 Jun 2016 19:02:17 -0400 Subject: Add --prune to fetch commands; track (old) HTTPS configuration changes; general cleanup. --- http-sync-repos.py | 87 ------------------------------- http-what-repos.py | 58 --------------------- https-sync-repos.py | 38 ++++++++++++++ https-what-repos.py | 14 +++++ ssh-sync-repos.py | 37 +++++++------ ssh-what-repos.py | 13 ++--- verilog-integer.py | 146 ++++++++++++++++++++++++++-------------------------- what-branches.py | 10 ++-- 8 files changed, 155 insertions(+), 248 deletions(-) delete mode 100755 http-sync-repos.py delete mode 100755 http-what-repos.py create mode 100755 https-sync-repos.py create mode 100755 https-what-repos.py diff --git a/http-sync-repos.py b/http-sync-repos.py deleted file mode 100755 index 2a0ab04..0000000 --- a/http-sync-repos.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python - -# Synchronize Cryptech git repositories by scraping URLs from the -# automatically generated Trac page. Yes, we know too much about how -# the generation script and Trac format this page, c'est la vie. -# -# Python's built-in support for HTTPS is a bit primative, we could do -# better with something like PyCURL, but we want this to run with just -# the standard Python libraries, so we're stranded in kludge city. - -from urllib import urlopen -from xml.etree.ElementTree import ElementTree -from subprocess import check_call -from tempfile import NamedTemporaryFile -import sys -import os - -# URL for automatically generated Trac page listing repositories. - -trac_page = "https://wiki.cryptech.is/wiki/GitRepositories" - -# PEM-encoded CA certificate covering Cryptech web services. - -cacert = '''\ ------BEGIN CERTIFICATE----- -MIIERzCCAy+gAwIBAgIBADANBgkqhkiG9w0BAQUFADB1MQswCQYDVQQGEwJVUzEL -MAkGA1UECBMCTUExEDAOBgNVBAcTB1JlYWRpbmcxITAfBgNVBAoTGEdydW5jaHdl -YXRoZXIgQXNzb2NpYXRlczEkMCIGCSqGSIb3DQEJARYVcG9zdG1hc3RlckBoYWN0 -cm4ubmV0MCAXDTEwMDkyNTA1MzQ0MloYDzIxMDkwOTI1MDUzNDQyWjB1MQswCQYD -VQQGEwJVUzELMAkGA1UECBMCTUExEDAOBgNVBAcTB1JlYWRpbmcxITAfBgNVBAoT -GEdydW5jaHdlYXRoZXIgQXNzb2NpYXRlczEkMCIGCSqGSIb3DQEJARYVcG9zdG1h -c3RlckBoYWN0cm4ubmV0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA -wf723VEaWozuyJCytEgFkEMXDd17RTChb4RCpqFrlswbKEk4/Vw7IvBeLYNfl5F8 -1hx6ca7CU4qD+4tVkanJ7dJhT9a1jWPKjx/Xz8rpZBSDzgiDZNlgX4giDHjG3Ju8 -QcT9Elpx4SEBxeaVEl2AXXsShv1PrSpCt9WUdS+0NQkA7z3cXASNPOBFoITM37pC -21nuQuocpjFTaGXw5s3/FhVgcKSUHmQN5UdYK5N2w7pcn7UV2v92UFYSG9cEjQL5 -DfPqzJeLemzmUl+XBqSfY/gPCyJdAPOudflnFyptVv3BrmMQYEyJBeRnt2o5aYWd -q1w2wB2HPsEglm1K19gEoQIDAQABo4HfMIHcMAwGA1UdEwQFMAMBAf8wCwYDVR0P -BAQDAgEGMB0GA1UdDgQWBBQkrUZV1MeYpBcTnpsuT60UPRjCZjCBnwYDVR0jBIGX -MIGUgBQkrUZV1MeYpBcTnpsuT60UPRjCZqF5pHcwdTELMAkGA1UEBhMCVVMxCzAJ -BgNVBAgTAk1BMRAwDgYDVQQHEwdSZWFkaW5nMSEwHwYDVQQKExhHcnVuY2h3ZWF0 -aGVyIEFzc29jaWF0ZXMxJDAiBgkqhkiG9w0BCQEWFXBvc3RtYXN0ZXJAaGFjdHJu -Lm5ldIIBADANBgkqhkiG9w0BAQUFAAOCAQEAMeBzm4x6hlwbcrR8AcjSP4S/8+pW -4WsmkuKlZMyP/E4rc2DpVBi9WRnLtQoQ+FDU47VkG1D97i+HYx1Ky7W8oU4YB3Xr -ptoAenVmiQOVnbMluuQlxBa8JbOizt+BzL0NJEC6gHvCYT5SmFFsJgXBQZNPpRJb -cDzuilf3b6vCFtRL+hD75VFLkljQWUWt59cNPGhtBItsq3q72LZ2ftPgk9ufC2h3 -whlvaaZldSFGnPRcFgXYYUootSh0yUW2lIHn52s/tPRN6XXnrdz+cLUCCchSkPOi -Q9q/UFgaXUetsqlb56PINOrLZUpAfqC4xFBVVo2oI0ubFq173QlATQpn/w== ------END CERTIFICATE----- -''' - -try: - ssl_cert_file = NamedTemporaryFile("w") - ssl_cert_file.write(cacert) - ssl_cert_file.flush() - try: - os.fchmod(ssl_cert_file.fileno(), 0600) - except: - pass - os.environ["SSL_CERT_FILE"] = ssl_cert_file.name # OpenSSL looks for this - os.environ["GIT_SSL_CAINFO"] = ssl_cert_file.name # Git looks for this - - head = "https://git.cryptech.is/" - tail = ".git" - errs = 0 - - for elt in ElementTree(file = urlopen(trac_page)).iter("{http://www.w3.org/1999/xhtml}tt"): - if elt.text.startswith(head) and elt.text.endswith(tail): - url = elt.text - repo = url[len(head):-len(tail)] - pull = os.path.isdir(repo) - print - print url - try: - if pull: - check_call(("git", "fetch", "--all"), cwd = repo) - check_call(("git", "pull"), cwd = repo) - else: - check_call(("git", "clone", url, repo)) - except: - print "** Error", "pulling" if pull else "cloning", repo - errs += 1 - -finally: - ssl_cert_file.close() - -sys.exit(errs) diff --git a/http-what-repos.py b/http-what-repos.py deleted file mode 100755 index 27125bd..0000000 --- a/http-what-repos.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python - -# List Cryptech git repository URLs by scraping them from the -# automatically generated Trac page. Yes, we know too much about how -# the generation script and Trac format this page, c'est la vie. - -from urllib import urlopen -from xml.etree.ElementTree import ElementTree -from tempfile import NamedTemporaryFile -from os import environ, fchmod - -url = "https://wiki.cryptech.is/wiki/GitRepositories" - -cacert = '''\ ------BEGIN CERTIFICATE----- -MIIERzCCAy+gAwIBAgIBADANBgkqhkiG9w0BAQUFADB1MQswCQYDVQQGEwJVUzEL -MAkGA1UECBMCTUExEDAOBgNVBAcTB1JlYWRpbmcxITAfBgNVBAoTGEdydW5jaHdl -YXRoZXIgQXNzb2NpYXRlczEkMCIGCSqGSIb3DQEJARYVcG9zdG1hc3RlckBoYWN0 -cm4ubmV0MCAXDTEwMDkyNTA1MzQ0MloYDzIxMDkwOTI1MDUzNDQyWjB1MQswCQYD -VQQGEwJVUzELMAkGA1UECBMCTUExEDAOBgNVBAcTB1JlYWRpbmcxITAfBgNVBAoT -GEdydW5jaHdlYXRoZXIgQXNzb2NpYXRlczEkMCIGCSqGSIb3DQEJARYVcG9zdG1h -c3RlckBoYWN0cm4ubmV0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA -wf723VEaWozuyJCytEgFkEMXDd17RTChb4RCpqFrlswbKEk4/Vw7IvBeLYNfl5F8 -1hx6ca7CU4qD+4tVkanJ7dJhT9a1jWPKjx/Xz8rpZBSDzgiDZNlgX4giDHjG3Ju8 -QcT9Elpx4SEBxeaVEl2AXXsShv1PrSpCt9WUdS+0NQkA7z3cXASNPOBFoITM37pC -21nuQuocpjFTaGXw5s3/FhVgcKSUHmQN5UdYK5N2w7pcn7UV2v92UFYSG9cEjQL5 -DfPqzJeLemzmUl+XBqSfY/gPCyJdAPOudflnFyptVv3BrmMQYEyJBeRnt2o5aYWd -q1w2wB2HPsEglm1K19gEoQIDAQABo4HfMIHcMAwGA1UdEwQFMAMBAf8wCwYDVR0P -BAQDAgEGMB0GA1UdDgQWBBQkrUZV1MeYpBcTnpsuT60UPRjCZjCBnwYDVR0jBIGX -MIGUgBQkrUZV1MeYpBcTnpsuT60UPRjCZqF5pHcwdTELMAkGA1UEBhMCVVMxCzAJ -BgNVBAgTAk1BMRAwDgYDVQQHEwdSZWFkaW5nMSEwHwYDVQQKExhHcnVuY2h3ZWF0 -aGVyIEFzc29jaWF0ZXMxJDAiBgkqhkiG9w0BCQEWFXBvc3RtYXN0ZXJAaGFjdHJu -Lm5ldIIBADANBgkqhkiG9w0BAQUFAAOCAQEAMeBzm4x6hlwbcrR8AcjSP4S/8+pW -4WsmkuKlZMyP/E4rc2DpVBi9WRnLtQoQ+FDU47VkG1D97i+HYx1Ky7W8oU4YB3Xr -ptoAenVmiQOVnbMluuQlxBa8JbOizt+BzL0NJEC6gHvCYT5SmFFsJgXBQZNPpRJb -cDzuilf3b6vCFtRL+hD75VFLkljQWUWt59cNPGhtBItsq3q72LZ2ftPgk9ufC2h3 -whlvaaZldSFGnPRcFgXYYUootSh0yUW2lIHn52s/tPRN6XXnrdz+cLUCCchSkPOi -Q9q/UFgaXUetsqlb56PINOrLZUpAfqC4xFBVVo2oI0ubFq173QlATQpn/w== ------END CERTIFICATE----- -''' - -try: - ssl_cert_file = NamedTemporaryFile("w") - ssl_cert_file.write(cacert) - ssl_cert_file.flush() - try: - fchmod(ssl_cert_file.fileno(), 0600) - except: - pass - environ["SSL_CERT_FILE"] = ssl_cert_file.name # OpenSSL looks for this - environ["GIT_SSL_CAINFO"] = ssl_cert_file.name # Git looks for this - - for x in ElementTree(file = urlopen(url)).iter("{http://www.w3.org/1999/xhtml}tt"): - if x.text.startswith("https://git.cryptech.is/") and x.text.endswith(".git"): - print x.text - -finally: - ssl_cert_file.close() diff --git a/https-sync-repos.py b/https-sync-repos.py new file mode 100755 index 0000000..ff1dd90 --- /dev/null +++ b/https-sync-repos.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# Synchronize Cryptech git repositories by scraping URLs from the +# automatically generated Trac page. Yes, we know too much about how +# the generation script and Trac format this page, c'est la vie. + +from urllib import urlopen +from xml.etree.ElementTree import ElementTree +from subprocess import check_call +from os.path import isdir +from sys import exit + +# URL for automatically generated Trac page listing repositories. + +trac_page = "https://wiki.cryptech.is/wiki/GitRepositories" + +head = "https://git.cryptech.is/" +tail = ".git" +errs = 0 + +for elt in ElementTree(file = urlopen(trac_page)).iter("{http://www.w3.org/1999/xhtml}tt"): + if elt.text.startswith(head) and elt.text.endswith(tail): + url = elt.text + repo = url[len(head):-len(tail)] + pull = isdir(repo) + print + print url + try: + if pull: + check_call(("git", "fetch", "--all", "--prune"), cwd = repo) + check_call(("git", "pull"), cwd = repo) + else: + check_call(("git", "clone", url, repo)) + except: + print "** Error", "pulling" if pull else "cloning", repo + errs += 1 + +exit(errs) diff --git a/https-what-repos.py b/https-what-repos.py new file mode 100755 index 0000000..5b66f19 --- /dev/null +++ b/https-what-repos.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + +# List Cryptech git repository URLs by scraping them from the +# automatically generated Trac page. Yes, we know too much about how +# the generation script and Trac format this page, c'est la vie. + +from urllib import urlopen +from xml.etree.ElementTree import ElementTree + +url = "https://wiki.cryptech.is/wiki/GitRepositories" + +for x in ElementTree(file = urlopen(url)).iter("{http://www.w3.org/1999/xhtml}tt"): + if x.text.startswith("https://git.cryptech.is/") and x.text.endswith(".git"): + print x.text diff --git a/ssh-sync-repos.py b/ssh-sync-repos.py index 5d585a3..fd418e4 100755 --- a/ssh-sync-repos.py +++ b/ssh-sync-repos.py @@ -3,26 +3,29 @@ # Synchronize Cryptech git repositories via SSH by asking gitolite for a JSON # listing of repositories. Not useful unless you have an SSH account, sorry. -import json, subprocess, os.path, sys +from subprocess import check_call, check_output +from os.path import isdir +from json import loads +from sys import exit user = "git@git.cryptech.is" -cmd = "ssh %s info -json -lc" % user -info = json.loads(subprocess.check_output(cmd.split())) +cmd = "ssh {} info -json -lc".format(user) +info = loads(check_output(cmd.split())) errs = 0 for repo in sorted(info["repos"]): - try: - if all(c not in repo for c in "*?[]"): - print - print repo - pull = os.path.isdir(repo) - if pull: - subprocess.check_call(("git", "fetch", "--all"), cwd = repo) - subprocess.check_call(("git", "pull"), cwd = repo) - else: - subprocess.check_call(("git", "clone", "%s:%s.git" % (user, repo), repo)) - except: - print "** Error", "pulling" if pull else "cloning", repo - errs += 1 + try: + if all(c not in repo for c in "*?[]"): + print + print repo + pull = isdir(repo) + if pull: + check_call(("git", "fetch", "--all", "--prune"), cwd = repo) + check_call(("git", "pull"), cwd = repo) + else: + check_call(("git", "clone", "{}:{}.git".format(user, repo), repo)) + except: + print "** Error", "pulling" if pull else "cloning", repo + errs += 1 -sys.exit(errs) +exit(errs) diff --git a/ssh-what-repos.py b/ssh-what-repos.py index 3e09720..6ecf6ed 100755 --- a/ssh-what-repos.py +++ b/ssh-what-repos.py @@ -4,12 +4,9 @@ # listing. Not useful unless you have an SSH account, sorry, and not # really all that much more interesting than normal gitolite output. -import json, subprocess +from json import loads +from subprocess import check_output -cmd = "ssh git@git.cryptech.is info -json -lc" - -info = json.loads(subprocess.check_output(cmd.split())) - -for name in sorted(info["repos"]): - if name != "gitolite-admin" and not any(c in name for c in "*?[]"): - print name +for repo in sorted(loads(check_output("ssh git@git.cryptech.is info -json -lc".split()))["repos"]): + if repo != "gitolite-admin" and not any(c in repo for c in "*?[]"): + print repo diff --git a/verilog-integer.py b/verilog-integer.py index e9a1ba5..5186f44 100644 --- a/verilog-integer.py +++ b/verilog-integer.py @@ -7,80 +7,80 @@ class VerilogInteger(object): - radix = dict(b = 2, o = 8, d = 10, h = 16) - - def __init__(self, input): - head, sep, tail = input.lower().translate(None, " \t_").partition("'") - self.input = input - self.code = tail[0] if tail else None - - if not sep: - self.width = 32 - self.value = int(head) - - elif self.code in self.radix: - self.width = int(head) if head else 32 - self.value = int(tail[1:], self.radix[self.code]) - if self.width <= 0 or self.value < 0: - raise ValueError - - else: - raise ValueError - - if self.width is not None: - mask = (1L << self.width) - 1 - if self.value > mask: - self.value &= mask - elif self.value < -mask: - self.value = -( -self.value & mask) - - @property - def C(self): - if self.code is None: - return str(self.value) - elif self.code == "d": - return "{0:d}".format(self.value) - elif self.code == "o": - return "0{0:0{1}o}".format(self.value, (self.width + 2) / 3) - else: - return "0x{0:0{1}x}".format(self.value, (self.width + 3) / 4) - - @property - def Verilog(self): - if self.code is None: - return str(self.value) - else: - fmt = "x" if self.code == "h" else self.code - return "{0.width}'{0.code}{0.value:{1}}".format(self, fmt) + radix = dict(b = 2, o = 8, d = 10, h = 16) + + def __init__(self, input): + head, sep, tail = input.lower().translate(None, " \t_").partition("'") + self.input = input + self.code = tail[0] if tail else None + + if not sep: + self.width = 32 + self.value = int(head) + + elif self.code in self.radix: + self.width = int(head) if head else 32 + self.value = int(tail[1:], self.radix[self.code]) + if self.width <= 0 or self.value < 0: + raise ValueError + + else: + raise ValueError + + if self.width is not None: + mask = (1L << self.width) - 1 + if self.value > mask: + self.value &= mask + elif self.value < -mask: + self.value = -( -self.value & mask) + + @property + def C(self): + if self.code is None: + return str(self.value) + elif self.code == "d": + return "{0:d}".format(self.value) + elif self.code == "o": + return "0{0:0{1}o}".format(self.value, (self.width + 2) / 3) + else: + return "0x{0:0{1}x}".format(self.value, (self.width + 3) / 4) + + @property + def Verilog(self): + if self.code is None: + return str(self.value) + else: + fmt = "x" if self.code == "h" else self.code + return "{0.width}'{0.code}{0.value:{1}}".format(self, fmt) if __name__ == "__main__": - def show(*args): - print "{:20} | {:20} | {:20}".format(*args) - - show("C", "Verilog", "Input") - show("-" * 20, "-" * 20, "-" * 20) - - def test(x): - v = VerilogInteger(x) - show(v.C, v.Verilog, v.input) - - test("15") - test("'h f") - test("'o 17") - test("'d 15") - test("'b 1111") - test("'b 1_1_1_1") - test("10 'd 20") - test("6'o 71") - test("8'b0") - test("8'b00000000") - test("8'b1") - test("8'b00000001") - - try: - for line in open("/tmp/sample-verilog-numbers"): - test(line.strip()) - except IOError: - pass + def show(*args): + print "{:20} | {:20} | {:20}".format(*args) + + show("C", "Verilog", "Input") + show("-" * 20, "-" * 20, "-" * 20) + + def test(x): + v = VerilogInteger(x) + show(v.C, v.Verilog, v.input) + + test("15") + test("'h f") + test("'o 17") + test("'d 15") + test("'b 1111") + test("'b 1_1_1_1") + test("10 'd 20") + test("6'o 71") + test("8'b0") + test("8'b00000000") + test("8'b1") + test("8'b00000001") + + try: + for line in open("/tmp/sample-verilog-numbers"): + test(line.strip()) + except IOError: + pass diff --git a/what-branches.py b/what-branches.py index 203b3d2..56e2082 100755 --- a/what-branches.py +++ b/what-branches.py @@ -7,8 +7,8 @@ from os import walk, listdir from sys import argv for root in argv[1:] or listdir("."): - for head, dirs, files in walk(root): - if ".git" in dirs and not head.endswith("/gitolite"): - print head - for line in check_output(("git", "branch", "-a"), cwd = head).splitlines(): - print " ", line + for head, dirs, files in walk(root): + if ".git" in dirs and not head.endswith("/gitolite"): + print head + for line in check_output(("git", "branch", "-a"), cwd = head).splitlines(): + print " ", line -- cgit v1.2.3