From b94f9f9d3816d3cd26a4cc8f3da9f4616bd05a35 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Wed, 14 Dec 2016 00:56:24 -0500 Subject: Support multiple packages corresponding to multiple releng branches. We want to be able to provide packaged builds of development branches. The most straightforward way to do this is a 1:1 correspondence between branches in the releng tree and variant package names. We adopt a simple convention: the base package name corresponds to the master branch, all other branches are named with the base package name followed by the branch name. So the master branch is the cryptech-alpha package, the ksng branch is the cryptech-alpha-ksng branch, and so forth. This isn't a perfect solution, but it's probably good enough. In order to do this, we need to generate the debian/control file at build-time, so that we can generate the list of conflicting packages. This commit also pulls in a few changes that had collected on the master branches of various repositories, chiefly because a few of them were necessary to get it the build to run at all. --- .gitignore | 1 + Makefile | 52 +++++++++++++----- build-firmware-package.py | 38 -------------- build-homebrew-formula.py | 95 --------------------------------- build-shadow-tree.py | 46 ---------------- scripts/build-debian-control-files.py | 58 ++++++++++++++++++++ scripts/build-firmware-package.py | 38 ++++++++++++++ scripts/build-homebrew-formula.py | 99 +++++++++++++++++++++++++++++++++++ scripts/build-shadow-tree.py | 46 ++++++++++++++++ source/debian/control | 22 -------- source/sw/libhal | 2 +- source/sw/stm32 | 2 +- 12 files changed, 282 insertions(+), 217 deletions(-) delete mode 100755 build-firmware-package.py delete mode 100755 build-homebrew-formula.py delete mode 100755 build-shadow-tree.py create mode 100755 scripts/build-debian-control-files.py create mode 100755 scripts/build-firmware-package.py create mode 100755 scripts/build-homebrew-formula.py create mode 100755 scripts/build-shadow-tree.py delete mode 100644 source/debian/control diff --git a/.gitignore b/.gitignore index 5a04258..ad15cf4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ cryptech-alpha_*_source.changes screenlog.* source/cryptech-alpha-firmware.tar.gz source/debian/changelog +source/debian/control tap diff --git a/Makefile b/Makefile index 17f6f10..3d4fde2 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,50 @@ # Top-level package build for Cryptech Alpha board. -PACKAGE_NAME := cryptech-alpha -PACKAGE_VERSION := 2.0.$(shell git show -s --format=%ct HEAD) +# What we call the package before we start mucking with branches and revision numbers + +PACKAGE_BASE_NAME := cryptech-alpha +PACKAGE_BASE_VERSION := 2.0 + +# Git voodoo: plumbing commands to pull the current branch and list of +# all (local) branches, and to pull something we can use as a version +# number suffix. +# +# Using a timestamp here is not particularly friendly, but we're +# looking for something simple that all the packaging systems involved +# are willing to accept as a version number, so, at least for now, we +# avoid more interesting options such as git-describe. + +GIT_VERSION := $(shell git show -s --format=%ct HEAD) +GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) +GIT_BRANCHES := $(notdir $(shell git for-each-ref --format '%(refname)' refs/heads/)) + +# Make voodoo: construct the package name, version number, and list of +# other package names (constructed on other branches) with which this +# one conflicts. + +PACKAGE_BRANCH = ${PACKAGE_BASE_NAME}$(and $(filter-out master,$(1)),-$(1)) +PACKAGE_NAME := $(call PACKAGE_BRANCH,${GIT_BRANCH}) +PACKAGE_CONFLICT := $(foreach I,$(filter-out ${GIT_BRANCH},${GIT_BRANCHES}),$(call PACKAGE_BRANCH,${I})) +PACKAGE_VERSION := ${PACKAGE_BASE_VERSION}.${GIT_VERSION} + +# gpg setup, for signing packages and repositories export GNUPGHOME := /home/aptbot/gnupg GPG_USER := APT Builder Robot GPG_KEYID := 37A8E93F5D7E7B9A +# Package repository setup + REPO_BASE := /home/aptbot REPO_UMASK := 002 +# Debian clean-room package builder setup + PBUILDER_BASE := ${HOME}/pbuilder PBUILDER_TARGETS := debian/jessie/i386 debian/jessie/amd64 ubuntu/xenial/i386 ubuntu/xenial/amd64 +# Where we upload the final results (if we do) + REPO_UPLOAD_USER := aptbot REPO_UPLOAD_HOST := bikeshed.cryptech.is REPO_UPLOAD_DIRS := apt brew @@ -28,14 +60,6 @@ BITSTREAM := build/core/platform/alpha/build/alpha_fmc.bit ELVES := build/sw/stm32/projects/bootloader/bootloader.elf build/sw/stm32/projects/hsm/hsm.elf TAMPER := build/sw/tamper/tamper.hex -# Command to generate a new changelog containing one entry. -# Does nothing if the changelog already exists. - -DCH = test -f debian/changelog || \ - EDITOR=true VISUAL=true TZ=UTC DEBEMAIL='${GPG_USER}' \ - dch --create --package ${PACKAGE_NAME} --newversion '${PACKAGE_VERSION}' \ - 'Software and firmware for Cryptech Alpha development board.' - all: init firmware dsc pbuilder homebrew expire enchilada: all upload @@ -54,10 +78,10 @@ sandblast: clean firmware: shadow ${FIRMWARE_TARBALL} shadow: - ./build-shadow-tree.py + ./scripts/build-shadow-tree.py ${FIRMWARE_TARBALL}: ${BITSTREAM} $(sort ${ELVES} ${ELVES:.elf=.bin}) ${TAMPER} - fakeroot ./build-firmware-package.py $@ $^ + fakeroot ./scripts/build-firmware-package.py $@ $^ bitstream: ${BITSTREAM} @@ -76,7 +100,7 @@ tamper: dsc: rm -f source/debian/changelog ${PACKAGE_NAME}_*.dsc ${PACKAGE_NAME}_*.tar.xz ${PACKAGE_NAME}_*_source.build ${PACKAGE_NAME}_*_source.changes - cd source; ${DCH} + cd source; ../scripts/build-debian-control-files.py --debemail='${GPG_USER}' --package ${PACKAGE_NAME} --newversion '${PACKAGE_VERSION}' --conflicts='${PACKAGE_CONFLICT}' cd source; debuild -S -uc -us pbuilder: @@ -95,7 +119,7 @@ homebrew: umask ${REPO_UMASK}; \ git clone ${REPO_BASE}/brew/tap tap; \ cd tap; \ - ../build-homebrew-formula.py ${REPO_BASE}/brew/tarballs/${PACKAGE_NAME}_${PACKAGE_VERSION}.tar.xz ${PACKAGE_VERSION} ${PACKAGE_NAME}.rb; \ + ../scripts/build-homebrew-formula.py ${REPO_BASE}/brew/tarballs/${PACKAGE_NAME}_${PACKAGE_VERSION}.tar.xz ${PACKAGE_VERSION} ${PACKAGE_NAME}.rb ${PACKAGE_CONFLICT}; \ git add ${PACKAGE_NAME}.rb; \ git commit -S${GPG_KEYID} --author='${GPG_USER}' -m '${PACKAGE_NAME} ${PACKAGE_VERSION}'; \ git push diff --git a/build-firmware-package.py b/build-firmware-package.py deleted file mode 100755 index c44b8bd..0000000 --- a/build-firmware-package.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import tempfile -import argparse -import hashlib -import tarfile -import json -import os - -parser = argparse.ArgumentParser() -parser.add_argument("tarfile", type = argparse.FileType("wb"), help = "tarball to create") -parser.add_argument("firmware", nargs = "+", help = "firmware files to stuff into tarball") -args = parser.parse_args() - -tar = tarfile.TarFile.open(fileobj = args.tarfile, mode = "w|gz") -head = subprocess.check_output(("git", "rev-parse", "HEAD")).strip() -time = subprocess.check_output(("git", "show", "-s", "--format=%ct", "HEAD")).strip() -commits = [line.split() for line in subprocess.check_output(("git", "submodule", "status")).splitlines()] -sha256 = {} - -for fn in args.firmware: - with open(fn, "rb") as f: - sha256[os.path.basename(fn)] = hashlib.sha256(f.read()).hexdigest() - tar.add(fn, os.path.basename(fn)) - -with tempfile.NamedTemporaryFile() as f: - os.fchmod(f.fileno(), 0644) - gpg = subprocess.Popen(("gpg", "--clearsign", "--personal-digest-preferences", "SHA256", "--no-permission-warning"), - stdin = subprocess.PIPE, stdout = f) - json.dump(dict(head = head, time = time, commits = commits, sha256 = sha256), gpg.stdin, indent = 2) - gpg.stdin.close() - if gpg.wait(): - raise subprocess.CalledProcessError(gpg.returncode, "gpg") - tar.add(f.name, "MANIFEST") - -tar.close() -args.tarfile.close() diff --git a/build-homebrew-formula.py b/build-homebrew-formula.py deleted file mode 100755 index f8adb7b..0000000 --- a/build-homebrew-formula.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python - -# Yes, this is a Python program writing a Ruby program. - -import argparse -import hashlib -import sys -import os - -parser = argparse.ArgumentParser() -parser.add_argument("--url-base", default = "https://brew.cryptech.is/tarballs/") -parser.add_argument("tarball") -parser.add_argument("version") -parser.add_argument("formula", type = argparse.FileType("w"), nargs = "?", default = sys.stdout) -args = parser.parse_args() - -template = '''\ -# This Homebrew forumula was automatically generated by a script. -# You might not want to edit it manually. - -class CryptechAlpha < Formula - - desc "Software for working with Cryptech Alpha board HSM" - homepage "https://cryptech.is/" - version "{version}" - url "{url}" - sha256 "{sha256}" - - # See https://github.com/Homebrew/brew/blob/master/share/doc/homebrew/Formula-Cookbook.md#specifying-other-formulae-as-dependencies - # for details on handling dependencies on other homebrew packages (eg, sqlite3). - - # See https://github.com/Homebrew/brew/blob/master/share/doc/homebrew/Python-for-Formula-Authors.md - # for details on handling dependencies on Python libraries (eg, pyserial). - - depends_on "sqlite3" - - resource "pyserial" do - url "https://pypi.python.org/packages/3c/d8/a9fa247ca60b02b3bebbd61766b4f321393b57b13c53b18f6f62cf172c08/pyserial-3.1.1.tar.gz" - sha256 "d657051249ce3cbd0446bcfb2be07a435e1029da4d63f53ed9b4cdde7373364c" - end - - resource "PyYAML" do - url "http://pyyaml.org/download/pyyaml/PyYAML-3.11.tar.gz" - sha256 "c36c938a872e5ff494938b33b14aaa156cb439ec67548fcab3535bb78b0846e8" - end - - def install - - # Installation is a bit complex due to the way Homebrew handles - # Python library dependencies and due to our stuff being a mix of - # Python and C. - - # Set PYTHONPATH to point to our private library location. - - ENV.prepend_create_path "PYTHONPATH", libexec/"vendor/lib/python2.7/site-packages" - - # Add all resources (and assume they are all Python, be careful...). - - resources.each do |r| - r.stage do - system "python", *Language::Python.setup_install_args(libexec/"vendor") - end - end - - # Build everything. - - ohai "Building PKCS #11 code (including crypto and bignum libraries) from source, this is slow, please be patient..." - ENV.deparallelize - system "make", "-C", "sw/pkcs11" - - # Install the Python scripts, then replace them with stubs which - # set PYTHONPATH before calling the real scripts. - - bin.install "sw/stm32/projects/hsm/cryptech_upload" - bin.install "sw/stm32/projects/hsm/cryptech_probe" - bin.install "sw/stm32/projects/hsm/cryptech_miniterm" - bin.env_script_all_files(libexec/"bin", :PYTHONPATH => ENV["PYTHONPATH"]) - - # Install other (non-Python) stuff, then we are done. - - share.install "cryptech-alpha-firmware.tar.gz" - lib.install "sw/pkcs11/libcryptech-pkcs11.dylib" - #bin.install "sw/pkcs11/p11util" - end - -end -''' - -with open(args.tarball, "rb") as f: - digest = hashlib.sha256(f.read()).hexdigest() - -args.formula.write(template.format( - version = args.version, - url = os.path.join(args.url_base, os.path.basename(args.tarball)), - sha256 = digest)) diff --git a/build-shadow-tree.py b/build-shadow-tree.py deleted file mode 100755 index 378797f..0000000 --- a/build-shadow-tree.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python - -# Create a symlink build tree like the old X11 project "lndir" program. -# -# Reason for this is simple: synthesizing the Alpha RTL code takes a -# looong time, so we don't want to do it unnecessarily, but we also -# don't want to include all of the intermediate files from the -# synthesis in the source tarball. So we symlink a shadow build tree -# off to the side, do the synthesis there. -# -# We could construct this symlink tree by hand, but that's fragile, so -# we'd probably write a script to do it anyway, so we might as well -# just use the script to build the shadow tree and have done with it. - -import os - -source_root = "source" -build_root = "build" - -if not os.path.isdir(build_root): - os.mkdir(build_root) - -for source_head, dirs, files in os.walk(source_root): - build_head = build_root + source_head[len(source_root):] - - for dn in dirs: - d = os.path.join(build_head, dn) - if not os.path.isdir(d): - os.mkdir(d) - - for fn in files: - if fn == ".git": - continue - d = os.path.join(build_head, fn) - s = os.path.join(source_head, fn) - s = os.path.abspath(s) - s = os.path.relpath(s, build_head) - if not os.path.islink(d): - os.symlink(s, d) - - for extra in set(os.listdir(build_head)) - set(dirs) - set(files): - d = os.path.join(build_head, extra) - if os.path.islink(d): - os.unlink(d) - elif os.path.isdir(d) and not os.listdir(d): - os.rmdir(d) diff --git a/scripts/build-debian-control-files.py b/scripts/build-debian-control-files.py new file mode 100755 index 0000000..1177049 --- /dev/null +++ b/scripts/build-debian-control-files.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +import subprocess +import argparse +import sys +import os + +parser = argparse.ArgumentParser() +parser.add_argument("--debemail", required = True) +parser.add_argument("--package", required = True) +parser.add_argument("--newversion", required = True) +parser.add_argument("--description", default = "Software and firmware for Cryptech Alpha development board.") +parser.add_argument("--conflicts", nargs = "*") + +args = parser.parse_args() + +if os.path.exists("debian/control") and os.path.exists("debian/changelog"): + sys.exit(0) + +control_template= '''\ +Source: {args.package} +Maintainer: {args.debemail} +Section: misc +Priority: optional +Standards-Version: 3.9.6 +Build-Depends: debhelper (>= 9), + dh-python, + libsqlite3-dev, + python (>= 2.7), + python-yaml +Homepage: http://trac.cryptech.is/wiki + +Package: cryptech-alpha +Architecture: any +Depends: python, + python-serial (>= 3.0), + ${{misc:Depends}}, + ${{python:Depends}}, + ${{shlibs:Depends}} +{conflicts}\ +Description: Cryptech Project open-source cryptographic software and firmware. + {args.description} +''' + +if args.conflicts: + conflicts = "Conflicts: {}\n".format(" ".join(args.conflicts)) +else: + conflicts = "" + +subprocess.check_call(("dch", "--create", "--package", args.package, "--newversion", args.newversion, args.description), + env = dict(os.environ, + EDITOR = "/bin/true", + VISUAL = "/bin/true", + TZ = "UTC", + DEBEMAIL = args.debemail)) + +with open("debian/control", "w") as f: + f.write(control_template.format(args = args, conflicts = conflicts)) diff --git a/scripts/build-firmware-package.py b/scripts/build-firmware-package.py new file mode 100755 index 0000000..c44b8bd --- /dev/null +++ b/scripts/build-firmware-package.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +import subprocess +import tempfile +import argparse +import hashlib +import tarfile +import json +import os + +parser = argparse.ArgumentParser() +parser.add_argument("tarfile", type = argparse.FileType("wb"), help = "tarball to create") +parser.add_argument("firmware", nargs = "+", help = "firmware files to stuff into tarball") +args = parser.parse_args() + +tar = tarfile.TarFile.open(fileobj = args.tarfile, mode = "w|gz") +head = subprocess.check_output(("git", "rev-parse", "HEAD")).strip() +time = subprocess.check_output(("git", "show", "-s", "--format=%ct", "HEAD")).strip() +commits = [line.split() for line in subprocess.check_output(("git", "submodule", "status")).splitlines()] +sha256 = {} + +for fn in args.firmware: + with open(fn, "rb") as f: + sha256[os.path.basename(fn)] = hashlib.sha256(f.read()).hexdigest() + tar.add(fn, os.path.basename(fn)) + +with tempfile.NamedTemporaryFile() as f: + os.fchmod(f.fileno(), 0644) + gpg = subprocess.Popen(("gpg", "--clearsign", "--personal-digest-preferences", "SHA256", "--no-permission-warning"), + stdin = subprocess.PIPE, stdout = f) + json.dump(dict(head = head, time = time, commits = commits, sha256 = sha256), gpg.stdin, indent = 2) + gpg.stdin.close() + if gpg.wait(): + raise subprocess.CalledProcessError(gpg.returncode, "gpg") + tar.add(f.name, "MANIFEST") + +tar.close() +args.tarfile.close() diff --git a/scripts/build-homebrew-formula.py b/scripts/build-homebrew-formula.py new file mode 100755 index 0000000..6d43b45 --- /dev/null +++ b/scripts/build-homebrew-formula.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +# Yes, this is a Python program writing a Ruby program. + +import argparse +import hashlib +import sys +import os + +parser = argparse.ArgumentParser() +parser.add_argument("--url-base", default = "https://brew.cryptech.is/tarballs/") +parser.add_argument("tarball") +parser.add_argument("version") +parser.add_argument("formula", type = argparse.FileType("w"), nargs = "?", default = sys.stdout) +parser.add_argument("conflicts", nargs = "*") +args = parser.parse_args() + +template = '''\ +# This Homebrew forumula was automatically generated by a script. +# You might not want to edit it manually. + +class CryptechAlpha < Formula + + desc "Software for working with Cryptech Alpha board HSM" + homepage "https://cryptech.is/" + version "{version}" + url "{url}" + sha256 "{sha256}" + +{conflicts} + + # See https://github.com/Homebrew/brew/blob/master/share/doc/homebrew/Formula-Cookbook.md#specifying-other-formulae-as-dependencies + # for details on handling dependencies on other homebrew packages (eg, sqlite3). + + # See https://github.com/Homebrew/brew/blob/master/share/doc/homebrew/Python-for-Formula-Authors.md + # for details on handling dependencies on Python libraries (eg, pyserial). + + depends_on "sqlite3" + + resource "pyserial" do + url "https://pypi.python.org/packages/3c/d8/a9fa247ca60b02b3bebbd61766b4f321393b57b13c53b18f6f62cf172c08/pyserial-3.1.1.tar.gz" + sha256 "d657051249ce3cbd0446bcfb2be07a435e1029da4d63f53ed9b4cdde7373364c" + end + + resource "PyYAML" do + url "http://pyyaml.org/download/pyyaml/PyYAML-3.11.tar.gz" + sha256 "c36c938a872e5ff494938b33b14aaa156cb439ec67548fcab3535bb78b0846e8" + end + + def install + + # Installation is a bit complex due to the way Homebrew handles + # Python library dependencies and due to our stuff being a mix of + # Python and C. + + # Set PYTHONPATH to point to our private library location. + + ENV.prepend_create_path "PYTHONPATH", libexec/"vendor/lib/python2.7/site-packages" + + # Add all resources (and assume they are all Python, be careful...). + + resources.each do |r| + r.stage do + system "python", *Language::Python.setup_install_args(libexec/"vendor") + end + end + + # Build everything. + + ohai "Building PKCS #11 code (including crypto and bignum libraries) from source, this is slow, please be patient..." + ENV.deparallelize + system "make", "-C", "sw/pkcs11" + + # Install the Python scripts, then replace them with stubs which + # set PYTHONPATH before calling the real scripts. + + bin.install "sw/stm32/projects/hsm/cryptech_upload" + bin.install "sw/stm32/projects/hsm/cryptech_probe" + bin.install "sw/stm32/projects/hsm/cryptech_miniterm" + bin.env_script_all_files(libexec/"bin", :PYTHONPATH => ENV["PYTHONPATH"]) + + # Install other (non-Python) stuff, then we are done. + + share.install "cryptech-alpha-firmware.tar.gz" + lib.install "sw/pkcs11/libcryptech-pkcs11.dylib" + #bin.install "sw/pkcs11/p11util" + end + +end +''' + +with open(args.tarball, "rb") as f: + digest = hashlib.sha256(f.read()).hexdigest() + +args.formula.write(template.format( + version = args.version, + url = os.path.join(args.url_base, os.path.basename(args.tarball)), + sha256 = digest, + conflicts = "".join("conflicts_with \"{}\", :because => \"firmware and pkcs11 library must match\"\n".format(i) for i in args.conflicts))) diff --git a/scripts/build-shadow-tree.py b/scripts/build-shadow-tree.py new file mode 100755 index 0000000..378797f --- /dev/null +++ b/scripts/build-shadow-tree.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +# Create a symlink build tree like the old X11 project "lndir" program. +# +# Reason for this is simple: synthesizing the Alpha RTL code takes a +# looong time, so we don't want to do it unnecessarily, but we also +# don't want to include all of the intermediate files from the +# synthesis in the source tarball. So we symlink a shadow build tree +# off to the side, do the synthesis there. +# +# We could construct this symlink tree by hand, but that's fragile, so +# we'd probably write a script to do it anyway, so we might as well +# just use the script to build the shadow tree and have done with it. + +import os + +source_root = "source" +build_root = "build" + +if not os.path.isdir(build_root): + os.mkdir(build_root) + +for source_head, dirs, files in os.walk(source_root): + build_head = build_root + source_head[len(source_root):] + + for dn in dirs: + d = os.path.join(build_head, dn) + if not os.path.isdir(d): + os.mkdir(d) + + for fn in files: + if fn == ".git": + continue + d = os.path.join(build_head, fn) + s = os.path.join(source_head, fn) + s = os.path.abspath(s) + s = os.path.relpath(s, build_head) + if not os.path.islink(d): + os.symlink(s, d) + + for extra in set(os.listdir(build_head)) - set(dirs) - set(files): + d = os.path.join(build_head, extra) + if os.path.islink(d): + os.unlink(d) + elif os.path.isdir(d) and not os.listdir(d): + os.rmdir(d) diff --git a/source/debian/control b/source/debian/control deleted file mode 100644 index fcca634..0000000 --- a/source/debian/control +++ /dev/null @@ -1,22 +0,0 @@ -Source: cryptech-alpha -Maintainer: APT Builder Robot -Section: misc -Priority: optional -Standards-Version: 3.9.6 -Build-Depends: debhelper (>= 9), - dh-python, - libsqlite3-dev, - python (>= 2.7), - python-yaml -Homepage: http://trac.cryptech.is/wiki - -Package: cryptech-alpha -Architecture: any -Depends: python, - python-serial (>= 3.0), - ${misc:Depends}, - ${python:Depends}, - ${shlibs:Depends} -Description: Cryptech open-source crypto software - "cryptech-alpha" contains software for use with the Cryptech Project - "Alpha" development board. diff --git a/source/sw/libhal b/source/sw/libhal index 0166b1b..1295f7e 160000 --- a/source/sw/libhal +++ b/source/sw/libhal @@ -1 +1 @@ -Subproject commit 0166b1b370862ab34335af3d5710304dc3546499 +Subproject commit 1295f7ebbfaff3ad098fe9d4cafa32a1f3750563 diff --git a/source/sw/stm32 b/source/sw/stm32 index 057c2bd..d172acb 160000 --- a/source/sw/stm32 +++ b/source/sw/stm32 @@ -1 +1 @@ -Subproject commit 057c2bd09138dfd626289b27929427021f1b1c2a +Subproject commit d172acba926b72c57c47697bd640c51c0fcb038d -- cgit v1.2.3