Support multiple packages corresponding to multiple releng branches.

We want to be able to provide packaged builds of development branches.
The most straightforward way to do this is a 1:1 correspondence
between branches in the releng tree and variant package names.

We adopt a simple convention: the base package name corresponds to the
master branch, all other branches are named with the base package name
followed by the branch name.  So the master branch is the
cryptech-alpha package, the ksng branch is the cryptech-alpha-ksng
branch, and so forth.  This isn't a perfect solution, but it's
probably good enough.

In order to do this, we need to generate the debian/control file at
build-time, so that we can generate the list of conflicting packages.

This commit also pulls in a few changes that had collected on the
master branches of various repositories, chiefly because a few of them
were necessary to get it the build to run at all.

4 files changed, 241 insertions, 0 deletions

diff --git a/scripts/build-debian-control-files.py b/scripts/build-debian-control-files.py
new file mode 100755
index 0000000..1177049
--- /dev/null
+++ b/scripts/build-debian-control-files.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+
+import subprocess
+import argparse
+import sys
+import os
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--debemail",    required = True)
+parser.add_argument("--package",     required = True)
+parser.add_argument("--newversion",  required = True)
+parser.add_argument("--description", default = "Software and firmware for Cryptech Alpha development board.")
+parser.add_argument("--conflicts",   nargs = "*")
+
+args = parser.parse_args()
+
+if os.path.exists("debian/control") and os.path.exists("debian/changelog"):
+    sys.exit(0)
+
+control_template= '''\
+Source: {args.package}
+Maintainer: {args.debemail}
+Section: misc
+Priority: optional
+Standards-Version: 3.9.6
+Build-Depends: debhelper (>= 9),
+               dh-python,
+               libsqlite3-dev,
+               python (>= 2.7),
+               python-yaml
+Homepage: http://trac.cryptech.is/wiki
+
+Package: cryptech-alpha
+Architecture: any
+Depends: python,
+         python-serial (>= 3.0),
+         ${{misc:Depends}},
+         ${{python:Depends}},
+         ${{shlibs:Depends}}
+{conflicts}\
+Description: Cryptech Project open-source cryptographic software and firmware.
+ {args.description}
+'''
+
+if args.conflicts:
+    conflicts = "Conflicts: {}\n".format(" ".join(args.conflicts))
+else:
+    conflicts = ""
+
+subprocess.check_call(("dch", "--create", "--package", args.package, "--newversion", args.newversion, args.description),
+                      env = dict(os.environ,
+                                 EDITOR   = "/bin/true",
+                                 VISUAL   = "/bin/true",
+                                 TZ       = "UTC",
+                                 DEBEMAIL = args.debemail))
+
+with open("debian/control", "w") as f:
+    f.write(control_template.format(args = args, conflicts = conflicts))
diff --git a/scripts/build-firmware-package.py b/scripts/build-firmware-package.py
new file mode 100755
index 0000000..c44b8bd
--- /dev/null
+++ b/scripts/build-firmware-package.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+import subprocess
+import tempfile
+import argparse
+import hashlib
+import tarfile
+import json
+import os
+
+parser = argparse.ArgumentParser()
+parser.add_argument("tarfile", type = argparse.FileType("wb"),  help = "tarball to create")
+parser.add_argument("firmware", nargs = "+",                    help = "firmware files to stuff into tarball")
+args = parser.parse_args()
+
+tar     = tarfile.TarFile.open(fileobj = args.tarfile, mode = "w|gz")
+head    = subprocess.check_output(("git", "rev-parse", "HEAD")).strip()
+time    = subprocess.check_output(("git", "show", "-s", "--format=%ct", "HEAD")).strip()
+commits = [line.split() for line in subprocess.check_output(("git", "submodule", "status")).splitlines()]
+sha256  = {}
+
+for fn in args.firmware:
+    with open(fn, "rb") as f:
+        sha256[os.path.basename(fn)] = hashlib.sha256(f.read()).hexdigest()
+    tar.add(fn, os.path.basename(fn))
+
+with tempfile.NamedTemporaryFile() as f:
+    os.fchmod(f.fileno(), 0644)
+    gpg = subprocess.Popen(("gpg", "--clearsign", "--personal-digest-preferences", "SHA256", "--no-permission-warning"),
+                           stdin = subprocess.PIPE, stdout = f)
+    json.dump(dict(head = head, time = time, commits = commits, sha256  = sha256), gpg.stdin, indent = 2)
+    gpg.stdin.close()
+    if gpg.wait():
+        raise subprocess.CalledProcessError(gpg.returncode, "gpg")
+    tar.add(f.name, "MANIFEST")
+
+tar.close()
+args.tarfile.close()
diff --git a/scripts/build-homebrew-formula.py b/scripts/build-homebrew-formula.py
new file mode 100755
index 0000000..6d43b45
--- /dev/null
+++ b/scripts/build-homebrew-formula.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+
+# Yes, this is a Python program writing a Ruby program.
+
+import argparse
+import hashlib
+import sys
+import os
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--url-base", default = "https://brew.cryptech.is/tarballs/")
+parser.add_argument("tarball")
+parser.add_argument("version")
+parser.add_argument("formula", type = argparse.FileType("w"), nargs = "?", default = sys.stdout)
+parser.add_argument("conflicts", nargs = "*")
+args = parser.parse_args()
+
+template = '''\
+# This Homebrew forumula was automatically generated by a script.
+# You might not want to edit it manually.
+
+class CryptechAlpha < Formula
+
+  desc     "Software for working with Cryptech Alpha board HSM"
+  homepage "https://cryptech.is/"
+  version  "{version}"
+  url      "{url}"
+  sha256   "{sha256}"
+
+{conflicts}
+
+  # See https://github.com/Homebrew/brew/blob/master/share/doc/homebrew/Formula-Cookbook.md#specifying-other-formulae-as-dependencies
+  # for details on handling dependencies on other homebrew packages (eg, sqlite3).
+
+  # See https://github.com/Homebrew/brew/blob/master/share/doc/homebrew/Python-for-Formula-Authors.md
+  # for details on handling dependencies on Python libraries (eg, pyserial).
+
+  depends_on "sqlite3"
+
+  resource "pyserial" do
+    url    "https://pypi.python.org/packages/3c/d8/a9fa247ca60b02b3bebbd61766b4f321393b57b13c53b18f6f62cf172c08/pyserial-3.1.1.tar.gz"
+    sha256 "d657051249ce3cbd0446bcfb2be07a435e1029da4d63f53ed9b4cdde7373364c"
+  end
+
+  resource "PyYAML" do
+    url    "http://pyyaml.org/download/pyyaml/PyYAML-3.11.tar.gz"
+    sha256 "c36c938a872e5ff494938b33b14aaa156cb439ec67548fcab3535bb78b0846e8"
+  end
+
+  def install
+
+    # Installation is a bit complex due to the way Homebrew handles
+    # Python library dependencies and due to our stuff being a mix of
+    # Python and C.
+
+    # Set PYTHONPATH to point to our private library location.
+
+    ENV.prepend_create_path "PYTHONPATH", libexec/"vendor/lib/python2.7/site-packages"
+
+    # Add all resources (and assume they are all Python, be careful...).
+
+    resources.each do |r|
+      r.stage do
+        system "python", *Language::Python.setup_install_args(libexec/"vendor")
+      end
+    end
+
+    # Build everything.
+
+    ohai "Building PKCS #11 code (including crypto and bignum libraries) from source, this is slow, please be patient..."
+    ENV.deparallelize
+    system "make", "-C", "sw/pkcs11"
+
+    # Install the Python scripts, then replace them with stubs which
+    # set PYTHONPATH before calling the real scripts.
+
+    bin.install   "sw/stm32/projects/hsm/cryptech_upload"
+    bin.install   "sw/stm32/projects/hsm/cryptech_probe"
+    bin.install   "sw/stm32/projects/hsm/cryptech_miniterm"
+    bin.env_script_all_files(libexec/"bin", :PYTHONPATH => ENV["PYTHONPATH"])
+
+    # Install other (non-Python) stuff, then we are done.
+
+    share.install "cryptech-alpha-firmware.tar.gz"
+    lib.install   "sw/pkcs11/libcryptech-pkcs11.dylib"
+    #bin.install  "sw/pkcs11/p11util"
+  end
+
+end
+'''
+
+with open(args.tarball, "rb") as f:
+    digest = hashlib.sha256(f.read()).hexdigest()
+
+args.formula.write(template.format(
+    version   = args.version,
+    url       = os.path.join(args.url_base, os.path.basename(args.tarball)),
+    sha256    = digest,
+    conflicts = "".join("conflicts_with \"{}\", :because => \"firmware and pkcs11 library must match\"\n".format(i) for i in args.conflicts)))
diff --git a/scripts/build-shadow-tree.py b/scripts/build-shadow-tree.py
new file mode 100755
index 0000000..378797f
--- /dev/null
+++ b/scripts/build-shadow-tree.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+
+# Create a symlink build tree like the old X11 project "lndir" program.
+#
+# Reason for this is simple: synthesizing the Alpha RTL code takes a
+# looong time, so we don't want to do it unnecessarily, but we also
+# don't want to include all of the intermediate files from the
+# synthesis in the source tarball.  So we symlink a shadow build tree
+# off to the side, do the synthesis there.
+#
+# We could construct this symlink tree by hand, but that's fragile, so
+# we'd probably write a script to do it anyway, so we might as well
+# just use the script to build the shadow tree and have done with it.
+
+import os
+
+source_root = "source"
+build_root  = "build"
+
+if not os.path.isdir(build_root):
+    os.mkdir(build_root)
+
+for source_head, dirs, files in os.walk(source_root):
+    build_head = build_root + source_head[len(source_root):]
+
+    for dn in dirs:
+        d = os.path.join(build_head, dn)
+        if not os.path.isdir(d):
+            os.mkdir(d)
+
+    for fn in files:
+        if fn == ".git":
+            continue
+        d = os.path.join(build_head,  fn)
+        s = os.path.join(source_head, fn)
+        s = os.path.abspath(s)
+        s = os.path.relpath(s, build_head)
+        if not os.path.islink(d):
+            os.symlink(s, d)
+
+    for extra in set(os.listdir(build_head)) - set(dirs) - set(files):
+        d = os.path.join(build_head, extra)
+        if os.path.islink(d):
+            os.unlink(d)
+        elif os.path.isdir(d) and not os.listdir(d):
+            os.rmdir(d)