Support reproducible XZ Linux archives without changing runtime contents

tools/linux_archive_smoke.py

@@ -25,7 +25,7 @@ def sha256(path):
 
 def extract_checked(archive_path, destination):
     from package_linux import safe_name
-    with tarfile.open(archive_path, "r:gz") as archive:
+    with tarfile.open(archive_path, "r:*") as archive:
         members = archive.getmembers()
         if not 1 <= len(members) <= 128 or len({item.name for item in members}) != len(members):
             raise ValueError("Invalid candidate member inventory")

tools/package_linux.py

@@ -8,6 +8,7 @@ import gzip
 import hashlib
 import io
 import json
+import lzma
 import os
 from pathlib import Path, PurePosixPath
 import re
@@ -169,7 +170,9 @@ def media_files(root, media):
     return selected
 
 
-def package(root, executable, runtime, media, output, llama_license=None):
+def package(root, executable, runtime, media, output, llama_license=None, compression="gzip"):
+    if compression not in {"gzip", "xz"}:
+        raise ValueError("Unsupported package compression")
     root, executable, runtime, media, output = map(Path, (root, executable, runtime, media, output))
     regular(executable, executable.parent)
     elf_x64(executable)
@@ -199,7 +202,7 @@ def package(root, executable, runtime, media, output, llama_license=None):
         temporary_path = Path(temporary.name)
     try:
         records = []
-        with temporary_path.open("wb") as raw, gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed, tarfile.open(fileobj=compressed, mode="w|") as archive:
+        with temporary_path.open("wb") as raw, (lzma.LZMAFile(raw, "w", preset=6) if compression == "xz" else gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0)) as compressed, tarfile.open(fileobj=compressed, mode="w|") as archive:
             for name, (path, expected_record) in sorted(selected.items()):
                 safe_name(name)
                 regular(path, path.parent)
@@ -249,9 +252,12 @@ def main():
     parser.add_argument("--runtime", type=Path, default=ROOT / ".searchi/runtime/auto")
     parser.add_argument("--media", type=Path, default=ROOT / ".searchi/media-runtime")
     parser.add_argument("--llama-license", type=Path)
-    parser.add_argument("--output", type=Path, default=ROOT / ".searchi/packages/searchi-linux-x86_64.tar.gz")
+    parser.add_argument("--compression", choices=["gzip", "xz"], default="gzip")
+    parser.add_argument("--output", type=Path)
     args = parser.parse_args()
-    package(ROOT, args.executable, args.runtime, args.media, args.output, args.llama_license)
+    if args.output is None:
+        args.output = ROOT / (".searchi/packages/searchi-linux-x86_64.tar.xz" if args.compression == "xz" else ".searchi/packages/searchi-linux-x86_64.tar.gz")
+    package(ROOT, args.executable, args.runtime, args.media, args.output, args.llama_license, args.compression)
     print(f"Packaged Linux x86_64 development candidate: {args.output}")
 
 

tools/test_package_linux.py

@@ -99,6 +99,22 @@ class LinuxPackageTests(unittest.TestCase):
                 self.run_package(fixture)
             self.assertEqual(previous, fixture[-1].read_bytes())
 
+    def test_xz_preserves_members_checksums_modes_and_reproducibility(self):
+        with tempfile.TemporaryDirectory() as directory:
+            fixture = self.fixture(directory)
+            self.run_package(fixture)
+            with tarfile.open(fixture[-1]) as archive:
+                original = {m.name: (archive.extractfile(m).read(), m.mode) for m in archive}
+            with mock.patch.object(pack, "dynamic", side_effect=self.metadata):
+                pack.package(*fixture, compression="xz")
+                first = fixture[-1].read_bytes()
+                pack.package(*fixture, compression="xz")
+            self.assertEqual(first, fixture[-1].read_bytes())
+            self.assertTrue(first.startswith(b"\xfd7zXZ\x00"))
+            with tarfile.open(fixture[-1]) as archive:
+                compressed = {m.name: (archive.extractfile(m).read(), m.mode) for m in archive}
+            self.assertEqual(original, compressed)
+
     def test_media_helper_tamper_is_rejected_against_pinned_archive(self):
         with tempfile.TemporaryDirectory() as directory:
             fixture = self.fixture(directory)