From 23eac4f149edc62bb7ccc36db623f9301e2d2fda Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Sun, 21 Jun 2020 15:27:45 -0400 Subject: Preliminary support for Python 3 This is a first step towards moving all of the Cryptech code from Python 2 to Python 3. At this stage, the goal is to make the same source code work in both language dialects, and to build packages which install both versions of the library code. This is a necessary step along the way, but since Python 2 is already past EOL as of this writing and since some distributions have started dropping all support for Python 2, we will almost certainly want to drop all Python 2 support in the relatively near future, if only because it's not really to do all the packaging right for both versions at once without much more trouble than a dead language dialect is probably worth. All platforms we care about should support Python 3 already, any that don't probably have much worse problems. So the primary purpose of pushing this particular commit is to archive what will probably be the last version supporting Python 2, while giving folks a chance to test the incoming Python 3 support a bit. Once we've cut loose from Python 2 for good, there's some cleanup we can and should do (eg, all the gymnastics to work around Python 2's handling of bytes as a form of text rather than a sequence of small integers), but for the moment we want to keep that compatability, albeit briefly. --- scripts/build-debian-control-files.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/build-debian-control-files.py b/scripts/build-debian-control-files.py index 9817248..c06055d 100755 --- a/scripts/build-debian-control-files.py +++ b/scripts/build-debian-control-files.py @@ -26,7 +26,8 @@ Standards-Version: 3.9.6 Build-Depends: debhelper (>= 9), dh-python, python (>= 2.7), - python-yaml + python-yaml, + python3 Homepage: http://trac.cryptech.is/wiki Package: {args.package} @@ -35,6 +36,10 @@ Depends: python, python-serial (>= 3.0), python-tornado (>= 4.0), python-crypto, + python3, + python3-serial (>= 3.0), + python3-tornado (>= 4.0), + python3-crypto, ${{misc:Depends}}, ${{python:Depends}}, ${{shlibs:Depends}} -- cgit v1.2.3 From 37329250395aa3ac724fca99bbd2de01efea10e1 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Sun, 21 Jun 2020 15:48:28 -0400 Subject: Typo in updated build script --- scripts/build-firmware-package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/build-firmware-package.py b/scripts/build-firmware-package.py index 55e77a7..9a58970 100755 --- a/scripts/build-firmware-package.py +++ b/scripts/build-firmware-package.py @@ -31,7 +31,7 @@ with tempfile.NamedTemporaryFile() as f: if use_gpg: gpg = subprocess.Popen(("gpg", "--clearsign", "--personal-digest-preferences", "SHA256", "--no-permission-warning"), stdin = subprocess.PIPE, stdout = f) - jf = gnupg.stdin + jf = gpg.stdin else: jf = f json.dump(dict(head = head, time = time, commits = commits, sha256 = sha256), jf, indent = 2) -- cgit v1.2.3 From 9e6b0d1117e21f2e6d092e337c5c522726016467 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Sun, 12 Jul 2020 18:25:56 -0400 Subject: Attempt to go Python3-only Ubuntu 20.04 no longer really supports Python 2, so we'd have to fork the packaging code if we wanted to keep support for Python 2 elsewhere. Given that Python 3 has been around for a more than a decade and that Python 2 was formally EOLed more than six months ago as of this writing, this seems like an unnecessary complication. The biggest change is rewriting the Homebrew formula for Python 3. --- scripts/build-debian-control-files.py | 13 ++++--------- scripts/build-firmware-package.py | 2 +- scripts/build-homebrew-formula.py | 14 +++++++++----- scripts/build-shadow-tree.py | 2 +- 4 files changed, 15 insertions(+), 16 deletions(-) (limited to 'scripts') diff --git a/scripts/build-debian-control-files.py b/scripts/build-debian-control-files.py index c06055d..46958ea 100755 --- a/scripts/build-debian-control-files.py +++ b/scripts/build-debian-control-files.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import subprocess import argparse @@ -25,23 +25,18 @@ Priority: optional Standards-Version: 3.9.6 Build-Depends: debhelper (>= 9), dh-python, - python (>= 2.7), - python-yaml, + python3-yaml, python3 Homepage: http://trac.cryptech.is/wiki Package: {args.package} Architecture: any -Depends: python, - python-serial (>= 3.0), - python-tornado (>= 4.0), - python-crypto, - python3, +Depends: python3, python3-serial (>= 3.0), python3-tornado (>= 4.0), python3-crypto, ${{misc:Depends}}, - ${{python:Depends}}, + ${{python3:Depends}}, ${{shlibs:Depends}} {conflicts}\ Description: Cryptech Project open-source cryptographic software and firmware. diff --git a/scripts/build-firmware-package.py b/scripts/build-firmware-package.py index 9a58970..659a63f 100755 --- a/scripts/build-firmware-package.py +++ b/scripts/build-firmware-package.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import subprocess import tempfile diff --git a/scripts/build-homebrew-formula.py b/scripts/build-homebrew-formula.py index 54cdcae..852b4b6 100755 --- a/scripts/build-homebrew-formula.py +++ b/scripts/build-homebrew-formula.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Yes, this is a Python program writing a Ruby program. @@ -32,7 +32,7 @@ template = '''\ # code, however, is what Homebrew considers "bindings", so we install # those where user scripts as well as our own can find them...then we # add a symlink so that our scripts can find our bindings regardless -# of which copy of Python 2.7 Homebrew decides we should use this week. +# of which copy of Python Homebrew decides we should use this week. # # We have to build our own software before installing our Python code, # because at least one of the Python modules we install @@ -54,6 +54,8 @@ class {classname} < Formula url "{url}" sha256 "{sha256}" + depends_on "python@3.8" + {conflicts} resource "pyserial" do @@ -88,11 +90,13 @@ class {classname} < Formula def install - ENV.prepend_create_path "PYTHONPATH", libexec/"vendor/lib/python2.7/site-packages" + xy = Language::Python.major_minor_version "python3" + + ENV.prepend_create_path "PYTHONPATH", libexec/"vendor/lib/python#{xy}/site-packages" resources.each do |r| r.stage do - system "python", *Language::Python.setup_install_args(libexec/"vendor") + system "python3", *Language::Python.setup_install_args(libexec/"vendor") end end @@ -103,7 +107,7 @@ class {classname} < Formula system "python", *Language::Python.setup_install_args(prefix) bin.env_script_all_files(libexec/"bin", :PYTHONPATH => ENV["PYTHONPATH"]) - ln_s lib/"python2.7/site-packages/cryptech", libexec/"vendor/lib/python2.7/site-packages/cryptech" + ln_s lib/"python#{xy}/site-packages/cryptech", libexec/"vendor/lib/python#{xy}/site-packages/cryptech" share.install "cryptech-alpha-firmware.tar.gz" lib.install "sw/pkcs11/libcryptech-pkcs11.dylib" diff --git a/scripts/build-shadow-tree.py b/scripts/build-shadow-tree.py index 378797f..0f3a4a1 100755 --- a/scripts/build-shadow-tree.py +++ b/scripts/build-shadow-tree.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Create a symlink build tree like the old X11 project "lndir" program. # -- cgit v1.2.3 From 8214b98bf00db2707eca7db9e60099097d032d59 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Sun, 12 Jul 2020 22:56:04 -0400 Subject: Syntax change for octal constants, sigh --- scripts/build-firmware-package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/build-firmware-package.py b/scripts/build-firmware-package.py index 659a63f..1a96fc7 100755 --- a/scripts/build-firmware-package.py +++ b/scripts/build-firmware-package.py @@ -26,7 +26,7 @@ for fn in args.firmware: tar.add(fn, os.path.basename(fn)) with tempfile.NamedTemporaryFile() as f: - os.fchmod(f.fileno(), 0644) + os.fchmod(f.fileno(), 0o644) use_gpg = os.path.isdir(os.getenv("GNUPGHOME", "")) if use_gpg: gpg = subprocess.Popen(("gpg", "--clearsign", "--personal-digest-preferences", "SHA256", "--no-permission-warning"), -- cgit v1.2.3 From 68f48b7e850de48063d95645c3d76e0a2be5c079 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Sun, 12 Jul 2020 23:14:47 -0400 Subject: More fun with str and bytes --- scripts/build-firmware-package.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/build-firmware-package.py b/scripts/build-firmware-package.py index 1a96fc7..4f078f4 100755 --- a/scripts/build-firmware-package.py +++ b/scripts/build-firmware-package.py @@ -14,10 +14,10 @@ parser.add_argument("firmware", nargs = "+", help = "firmware args = parser.parse_args() tar = tarfile.TarFile.open(fileobj = args.tarfile, mode = "w|gz") -head = subprocess.check_output(("git", "rev-parse", "HEAD")).strip() -time = subprocess.check_output(("git", "show", "-s", "--format=%ct", "HEAD")).strip() +head = subprocess.check_output(("git", "rev-parse", "HEAD")).strip().decode() +time = subprocess.check_output(("git", "show", "-s", "--format=%ct", "HEAD")).strip().decode() commits = dict((path, hash) for hash, path, branch in - (line.split() for line in subprocess.check_output(("git", "submodule", "status")).splitlines())) + (line.decode().split() for line in subprocess.check_output(("git", "submodule", "status")).splitlines())) sha256 = {} for fn in args.firmware: @@ -34,7 +34,7 @@ with tempfile.NamedTemporaryFile() as f: jf = gpg.stdin else: jf = f - json.dump(dict(head = head, time = time, commits = commits, sha256 = sha256), jf, indent = 2) + jf.write(json.dumps(dict(head = head, time = time, commits = commits, sha256 = sha256), indent = 2).encode()) if use_gpg: gpg.stdin.close() if gpg.wait(): -- cgit v1.2.3 From 4d6f6ceebcb0422bfcf3443e7f4eb7a9eb1e4338 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Mon, 13 Jul 2020 00:36:11 -0400 Subject: Still more fun building packages with Python 3 --- scripts/build-firmware-package.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) (limited to 'scripts') diff --git a/scripts/build-firmware-package.py b/scripts/build-firmware-package.py index 4f078f4..6ce49ab 100755 --- a/scripts/build-firmware-package.py +++ b/scripts/build-firmware-package.py @@ -6,6 +6,7 @@ import argparse import hashlib import tarfile import json +import sys import os parser = argparse.ArgumentParser() @@ -14,8 +15,8 @@ parser.add_argument("firmware", nargs = "+", help = "firmware args = parser.parse_args() tar = tarfile.TarFile.open(fileobj = args.tarfile, mode = "w|gz") -head = subprocess.check_output(("git", "rev-parse", "HEAD")).strip().decode() -time = subprocess.check_output(("git", "show", "-s", "--format=%ct", "HEAD")).strip().decode() +head = subprocess.check_output(("git", "rev-parse", "HEAD")).decode().strip() +time = subprocess.check_output(("git", "show", "-s", "--format=%ct", "HEAD")).decode().strip() commits = dict((path, hash) for hash, path, branch in (line.decode().split() for line in subprocess.check_output(("git", "submodule", "status")).splitlines())) sha256 = {} @@ -25,20 +26,18 @@ for fn in args.firmware: sha256[os.path.basename(fn)] = hashlib.sha256(f.read()).hexdigest() tar.add(fn, os.path.basename(fn)) -with tempfile.NamedTemporaryFile() as f: +manifest = json.dumps(dict(head = head, time = time, commits = commits, sha256 = sha256), indent = 2, sort_keys = True) + +if os.path.isdir(os.getenv("GNUPGHOME", "")): + gpg = subprocess.Popen(("gpg", "--clearsign", "--personal-digest-preferences", "SHA256", "--no-permission-warning"), + stdin = subprocess.PIPE, stdout = subprocess.PIPE, universal_newlines = True) + manifest = gpg.communicate(manifest)[0] + if gpg.returncode: + sys.exit("gpg failed") + +with tempfile.NamedTemporaryFile("w+") as f: os.fchmod(f.fileno(), 0o644) - use_gpg = os.path.isdir(os.getenv("GNUPGHOME", "")) - if use_gpg: - gpg = subprocess.Popen(("gpg", "--clearsign", "--personal-digest-preferences", "SHA256", "--no-permission-warning"), - stdin = subprocess.PIPE, stdout = f) - jf = gpg.stdin - else: - jf = f - jf.write(json.dumps(dict(head = head, time = time, commits = commits, sha256 = sha256), indent = 2).encode()) - if use_gpg: - gpg.stdin.close() - if gpg.wait(): - raise subprocess.CalledProcessError(gpg.returncode, "gpg") + f.write(manifest) f.seek(0) tar.add(f.name, "MANIFEST") -- cgit v1.2.3 From ed161c5f748ce4ffa382b78b4718fe1bc6a98a03 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Mon, 13 Jul 2020 01:26:43 -0400 Subject: Rewrite script to use subprocess.run(), another python -> python3 --- scripts/build-firmware-package.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) (limited to 'scripts') diff --git a/scripts/build-firmware-package.py b/scripts/build-firmware-package.py index 6ce49ab..b36b55e 100755 --- a/scripts/build-firmware-package.py +++ b/scripts/build-firmware-package.py @@ -9,16 +9,22 @@ import json import sys import os +def run(*args, **kwargs): + kwargs.update(stdout = subprocess.PIPE, universal_newlines = True, check = True) + return subprocess.run(args, **kwargs) + parser = argparse.ArgumentParser() parser.add_argument("tarfile", type = argparse.FileType("wb"), help = "tarball to create") parser.add_argument("firmware", nargs = "+", help = "firmware files to stuff into tarball") args = parser.parse_args() tar = tarfile.TarFile.open(fileobj = args.tarfile, mode = "w|gz") -head = subprocess.check_output(("git", "rev-parse", "HEAD")).decode().strip() -time = subprocess.check_output(("git", "show", "-s", "--format=%ct", "HEAD")).decode().strip() -commits = dict((path, hash) for hash, path, branch in - (line.decode().split() for line in subprocess.check_output(("git", "submodule", "status")).splitlines())) +head = run("git", "rev-parse", "HEAD").stdout.strip() +time = run("git", "show", "-s", "--format=%ct", "HEAD").stdout.strip() +commits = { path: hash + for hash, path, branch in ( + line.split() + for line in run("git", "submodule", "status").stdout.splitlines() ) } sha256 = {} for fn in args.firmware: @@ -29,11 +35,8 @@ for fn in args.firmware: manifest = json.dumps(dict(head = head, time = time, commits = commits, sha256 = sha256), indent = 2, sort_keys = True) if os.path.isdir(os.getenv("GNUPGHOME", "")): - gpg = subprocess.Popen(("gpg", "--clearsign", "--personal-digest-preferences", "SHA256", "--no-permission-warning"), - stdin = subprocess.PIPE, stdout = subprocess.PIPE, universal_newlines = True) - manifest = gpg.communicate(manifest)[0] - if gpg.returncode: - sys.exit("gpg failed") + manifest = run("gpg", "--clearsign", "--personal-digest-preferences", "SHA256", "--no-permission-warning", + input = manifest).stdout with tempfile.NamedTemporaryFile("w+") as f: os.fchmod(f.fileno(), 0o644) -- cgit v1.2.3 From 69ac7eaae12a51820f01280ad4a487a1c21fc1af Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Mon, 13 Jul 2020 13:18:16 -0400 Subject: Double-brace Ruby format voodoo to get past Python format vodooo. --- scripts/build-homebrew-formula.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/build-homebrew-formula.py b/scripts/build-homebrew-formula.py index 852b4b6..c14bfc5 100755 --- a/scripts/build-homebrew-formula.py +++ b/scripts/build-homebrew-formula.py @@ -92,7 +92,7 @@ class {classname} < Formula xy = Language::Python.major_minor_version "python3" - ENV.prepend_create_path "PYTHONPATH", libexec/"vendor/lib/python#{xy}/site-packages" + ENV.prepend_create_path "PYTHONPATH", libexec/"vendor/lib/python#{{xy}}/site-packages" resources.each do |r| r.stage do @@ -107,7 +107,7 @@ class {classname} < Formula system "python", *Language::Python.setup_install_args(prefix) bin.env_script_all_files(libexec/"bin", :PYTHONPATH => ENV["PYTHONPATH"]) - ln_s lib/"python#{xy}/site-packages/cryptech", libexec/"vendor/lib/python#{xy}/site-packages/cryptech" + ln_s lib/"python#{{xy}}/site-packages/cryptech", libexec/"vendor/lib/python#{{xy}}/site-packages/cryptech" share.install "cryptech-alpha-firmware.tar.gz" lib.install "sw/pkcs11/libcryptech-pkcs11.dylib" -- cgit v1.2.3 From 9cff51b9742504e54e90f1540dbd621da91ac9ee Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Tue, 1 Sep 2020 11:11:21 -0400 Subject: Debug Homebrew formula for Python3 --- scripts/build-homebrew-formula.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) (limited to 'scripts') diff --git a/scripts/build-homebrew-formula.py b/scripts/build-homebrew-formula.py index c14bfc5..56987c4 100755 --- a/scripts/build-homebrew-formula.py +++ b/scripts/build-homebrew-formula.py @@ -59,13 +59,13 @@ class {classname} < Formula {conflicts} resource "pyserial" do - url "https://pypi.python.org/packages/3c/d8/a9fa247ca60b02b3bebbd61766b4f321393b57b13c53b18f6f62cf172c08/pyserial-3.1.1.tar.gz" - sha256 "d657051249ce3cbd0446bcfb2be07a435e1029da4d63f53ed9b4cdde7373364c" + url "https://files.pythonhosted.org/packages/cc/74/11b04703ec416717b247d789103277269d567db575d2fd88f25d9767fe3d/pyserial-3.4.tar.gz" + sha256 "6e2d401fdee0eab996cf734e67773a0143b932772ca8b42451440cfed942c627" end resource "PyYAML" do - url "http://pyyaml.org/download/pyyaml/PyYAML-3.11.tar.gz" - sha256 "c36c938a872e5ff494938b33b14aaa156cb439ec67548fcab3535bb78b0846e8" + url "http://pyyaml.org/download/pyyaml/PyYAML-5.3.1.tar.gz" + sha256 "b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d" end resource "tornado" do @@ -73,16 +73,6 @@ class {classname} < Formula sha256 "f267acc96d5cf3df0fd8a7bfb5a91c2eb4ec81d5962d1a7386ceb34c655634a8" end - resource "singledispatch" do - url "https://pypi.python.org/packages/source/s/singledispatch/singledispatch-3.4.0.3.tar.gz" - sha256 "5b06af87df13818d14f08a028e42f566640aef80805c3b50c5056b086e3c2b9c" - end - - resource "backports_abc" do - url "https://files.pythonhosted.org/packages/source/b/backports_abc/backports_abc-0.5.tar.gz" - sha256 "033be54514a03e255df75c5aee8f9e672f663f93abb723444caec8fe43437bde" - end - resource "pycrypto" do url "https://pypi.python.org/packages/source/p/pycrypto/pycrypto-2.6.1.tar.gz" sha256 "f2ce1e989b272cfcb677616763e0a2e7ec659effa67a88aa92b3a65528f60a3c" @@ -95,6 +85,7 @@ class {classname} < Formula ENV.prepend_create_path "PYTHONPATH", libexec/"vendor/lib/python#{{xy}}/site-packages" resources.each do |r| + ohai "Building resource #{{r.url}}" r.stage do system "python3", *Language::Python.setup_install_args(libexec/"vendor") end @@ -104,7 +95,7 @@ class {classname} < Formula ENV.deparallelize system "make", "-C", "sw/pkcs11" - system "python", *Language::Python.setup_install_args(prefix) + system "python3", *Language::Python.setup_install_args(prefix) bin.env_script_all_files(libexec/"bin", :PYTHONPATH => ENV["PYTHONPATH"]) ln_s lib/"python#{{xy}}/site-packages/cryptech", libexec/"vendor/lib/python#{{xy}}/site-packages/cryptech" @@ -122,8 +113,8 @@ with open(args.tarball, "rb") as f: classname = "".join(word.capitalize() for word in args.package.split("-")) -conflicts = "".join(" conflicts_with \"{}\", :because => \"HSM firmware and PKCS #11 library must match\"\n".format(conflict) - for conflict in args.conflicts.split()) +conflicts = "\n".join(" conflicts_with \"{}\", :because => \"HSM firmware and PKCS #11 library must match\"".format(conflict) + for conflict in args.conflicts.split()) url = os.path.join(args.url_base, os.path.basename(args.tarball)) -- cgit v1.2.3