#!/usr/bin/env python3 """ autopackage.py This is a rewrite/port of my original autopackage python script, that i used to just keep to myself but... now im just trusting enough of my personal supply chain that i think im comfy sharing package builds... at least for friends! I mostly intend for CI to run this script but, if you want to make tweaks or experiment locally thats super fine too! Examples: ./autopackage.py --dry-run ./autopackage.py --only openrgb-git paru """ from __future__ import annotations import argparse import base64 import io import json import os import re import shlex import shutil import subprocess import sys import tarfile import time import urllib.error import urllib.request from dataclasses import dataclass, field from pathlib import Path from typing import List, Optional AUR_BASE = "https://aur.archlinux.org" DEFAULT_IGNORE = {".git", ".gitea", ".github", ".build", "scripts", "dist", "__pycache__"} PKG_GLOBS = ("*.pkg.tar.zst", "*.pkg.tar.xz", "*.pkg.tar.gz", "*.pkg.tar.bz2") PATCH_SUFFIXES = (".patch", ".diff") @dataclass class Config: repo_root: Path workdir: Path makepkg_cmd: List[str] sign: bool keep_going: bool dry_run: bool publish: bool replace: bool force: bool registry_url: str owner: str arch_repo: str auth_user: str token: str ignore: set = field(default_factory=set) def run(cmd: List[str], cwd: Optional[Path] = None, dry_run: bool = False, check: bool = True) -> subprocess.CompletedProcess: pretty = " ".join(shlex.quote(c) for c in cmd) if cwd: pretty = f"(cd {cwd}) {pretty}" print(f"+ {pretty}", flush=True) if dry_run: return subprocess.CompletedProcess(cmd, 0) return subprocess.run(cmd, cwd=str(cwd) if cwd else None, check=check) def env(name: str, default: str = "") -> str: return os.environ.get(name, default) def load_ignore(repo_root: Path) -> set: ignore = set(DEFAULT_IGNORE) path = repo_root / ".autopackageignore" if path.is_file(): for line in path.read_text(encoding="utf-8").splitlines(): line = line.strip() if line and not line.startswith("#"): ignore.add(line) return ignore def discover_packages(cfg: Config, only: List[str]) -> List[Path]: pkgs = [] for entry in sorted(cfg.repo_root.iterdir()): if not entry.is_dir() or entry.name.startswith(".") or entry.name in cfg.ignore: continue if only and entry.name not in only: continue pkgs.append(entry) return pkgs def package_files(pkg_src: Path) -> List[Path]: return sorted( p for p in pkg_src.iterdir() if p.is_file() and p.name != ".gitkeep" ) def is_custom(pkg_src: Path) -> bool: return (pkg_src / "PKGBUILD").is_file() def reset_build_dir(build_dir: Path, dry_run: bool) -> None: if dry_run: print(f"+ rm -rf {build_dir}") return if build_dir.exists(): shutil.rmtree(build_dir) build_dir.parent.mkdir(parents=True, exist_ok=True) def prepare_build_dir(pkg_src: Path, build_dir: Path, cfg: Config) -> None: name = pkg_src.name if is_custom(pkg_src): print(f" custom -> copy {pkg_src}") if cfg.dry_run: print(f"+ cp -r {pkg_src} {build_dir}") return shutil.copytree(pkg_src, build_dir, ignore=shutil.ignore_patterns(".git")) return print(f" AUR -> clone {AUR_BASE}/{name}.git") run(["git", "clone", "--depth", "1", f"{AUR_BASE}/{name}.git", str(build_dir)], dry_run=cfg.dry_run) patches: List[str] = [] source_patches: List[str] = [] for src in package_files(pkg_src): if src.suffix in PATCH_SUFFIXES: patches.append(src.name) print(f" patch: {src.name}") else: print(f" override: {src.name}") if cfg.dry_run: print(f"+ cp {src} {build_dir / src.name}") continue shutil.copy2(src, build_dir / src.name) if cfg.dry_run: if patches: print(f"+ apply packaging patches (if any), wire rest into PKGBUILD") return for patch in patches: path = build_dir / patch if apply_packaging_patch(path, build_dir): print(f" packaging patch applied: {patch}") else: source_patches.append(patch) if source_patches: wire_source_patches(build_dir / "PKGBUILD", source_patches) def apply_packaging_patch(patch: Path, build_dir: Path) -> bool: """Apply a patch to the AUR packaging tree (PKGBUILD etc.), if it fits.""" check = subprocess.run( ["git", "apply", "--check", "--whitespace=nowarn", str(patch)], cwd=str(build_dir), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) if check.returncode != 0: return False run(["git", "apply", "--whitespace=nowarn", str(patch)], cwd=build_dir) return True def wire_source_patches(pkgbuild: Path, patches: List[str]) -> None: """Append patches to source= and apply them in prepare() after the first cd.""" text = pkgbuild.read_text(encoding="utf-8") for patch in patches: if patch in text: continue text = text.rstrip() + f'\nsource+=("{patch}")\nsha256sums+=("SKIP")\n' m = re.search(r"(prepare\(\)\s*\{[^\n]*\n\s*cd [^\n]+\n)", text) if not m: raise RuntimeError( f"{pkgbuild}: need a prepare() with a cd line to apply {patch}" ) insert = f' patch -Np1 -i "$srcdir/{patch}"\n' text = text[:m.end()] + insert + text[m.end():] pkgbuild.write_text(text, encoding="utf-8") def find_artifacts(build_dir: Path, want_sig: bool) -> List[Path]: found: List[Path] = [] for pat in PKG_GLOBS: found.extend(build_dir.glob(pat)) pkgs = sorted( (p for p in found if p.is_file() and not p.name.endswith(".sig")), key=lambda p: p.stat().st_mtime, reverse=True, ) out: List[Path] = [] for p in pkgs: out.append(p) if want_sig: sig = p.with_name(p.name + ".sig") if sig.is_file(): out.append(sig) return out # --------------------------------------------------------------------------- # # Skip rebuild when this version is already on the registry # --------------------------------------------------------------------------- # _REGISTRY_FILES: Optional[set] = None def registry_base(cfg: Config) -> str: return f"{cfg.registry_url.rstrip('/')}/api/packages/{cfg.owner}/arch/{cfg.arch_repo}" def auth_header(cfg: Config) -> str: raw = f"{cfg.auth_user}:{cfg.token}".encode() return "Basic " + base64.b64encode(raw).decode("ascii") def http(url: str, method: str, cfg: Config, data: Optional[bytes] = None, retries: int = 4) -> tuple: last_err: Optional[Exception] = None for attempt in range(1, retries + 1): req = urllib.request.Request(url, data=data, method=method) if cfg.token: req.add_header("Authorization", auth_header(cfg)) if data is not None: req.add_header("Content-Type", "application/octet-stream") try: with urllib.request.urlopen(req, timeout=300) as resp: return resp.status, resp.read() except urllib.error.HTTPError as e: body = e.read() if e.code == 404: return 404, body if e.code in (500, 502, 503, 504) and attempt < retries: last_err = e else: return e.code, body except (urllib.error.URLError, ConnectionError, TimeoutError, OSError) as e: last_err = e if attempt >= retries: raise time.sleep(min(2 ** attempt, 30)) raise RuntimeError(f"{method} {url} failed: {last_err}") def registry_filenames(cfg: Config) -> Optional[set]: global _REGISTRY_FILES if _REGISTRY_FILES is not None: return _REGISTRY_FILES if not cfg.registry_url or not cfg.owner: return None names: set = set() ok = False for arch in ("x86_64", "any"): url = f"{registry_base(cfg)}/{arch}/{cfg.arch_repo}.db" try: status, data = http(url, "GET", cfg) except (RuntimeError, OSError): continue if status == 404: ok = True continue if status != 200 or not data: continue ok = True try: with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tf: for m in tf.getmembers(): if not m.isfile() or not m.name.endswith("/desc"): continue fh = tf.extractfile(m) if not fh: continue lines = fh.read().decode("utf-8", "replace").splitlines() for i, line in enumerate(lines): if line.strip() == "%FILENAME%" and i + 1 < len(lines): names.add(lines[i + 1].strip()) break except (tarfile.TarError, OSError): continue if not ok: return None _REGISTRY_FILES = names return names def srcinfo(build_dir: Path) -> Optional[dict]: pkgbuild = build_dir / "PKGBUILD" if not pkgbuild.is_file(): return None if re.search(r"^\s*pkgver\s*\(\)", pkgbuild.read_text(errors="replace"), re.M): return {"vcs": True} try: out = subprocess.run( ["makepkg", "--printsrcinfo"], cwd=build_dir, capture_output=True, text=True, check=True, ).stdout except (subprocess.CalledProcessError, OSError): return None info = {"vcs": False, "names": [], "pkgver": None, "pkgrel": None, "epoch": None, "arch": "x86_64"} for line in out.splitlines(): if "=" not in line: continue key, val = (p.strip() for p in line.split("=", 1)) if key == "pkgname": info["names"].append(val) elif key == "pkgver": info["pkgver"] = val elif key == "pkgrel": info["pkgrel"] = val elif key == "epoch": info["epoch"] = val elif key == "arch": if val == "x86_64": info["arch"] = "x86_64" elif val == "any" and info["arch"] != "x86_64": info["arch"] = "any" if not info["names"] or not info["pkgver"] or not info["pkgrel"]: return None ver = info["pkgver"] if info["epoch"]: ver = f"{info['epoch']}:{ver}" info["fullver"] = f"{ver}-{info['pkgrel']}" return info def already_built(build_dir: Path, cfg: Config) -> bool: info = srcinfo(build_dir) if not info or info.get("vcs"): return False published = registry_filenames(cfg) if published is None: return False arch = info["arch"] for name in info["names"]: stem = f"{name}-{info['fullver']}-{arch}.pkg.tar" if not any(f.startswith(stem) for f in published): return False return True # --------------------------------------------------------------------------- # # Publish # --------------------------------------------------------------------------- # def parse_pkg_filename(filename: str) -> Optional[tuple]: base = filename for suffix in (".pkg.tar.zst", ".pkg.tar.xz", ".pkg.tar.gz", ".pkg.tar.bz2"): if base.endswith(suffix): base = base[: -len(suffix)] break else: return None parts = base.rsplit("-", 3) if len(parts) != 4: return None name, ver, rel, arch = parts return name, f"{ver}-{rel}", arch def publish_file(path: Path, cfg: Config) -> None: url = registry_base(cfg) if cfg.dry_run: print(f"+ PUT {url} ({path.name})") return data = path.read_bytes() status, body = http(url, "PUT", cfg, data=data) body_s = body.decode("utf-8", "replace").strip()[:500] if status in (200, 201): print(f" uploaded {path.name} -> {status}") return if status == 409 and cfg.replace: meta = parse_pkg_filename(path.name) if meta: n, ver, arch = meta http(f"{registry_base(cfg)}/{n}/{ver}/{arch}", "DELETE", cfg) status, body = http(url, "PUT", cfg, data=data) body_s = body.decode("utf-8", "replace").strip()[:500] if status in (200, 201): print(f" uploaded {path.name} -> {status}") return if status == 409: print(f" {path.name} already published (409); skipping") return if status in (401, 403): raise RuntimeError( f"upload of {path.name} failed: HTTP {status}: {body_s}\n" f" -> need a PAT with write:package (PACKAGE_TOKEN), not github.token" ) raise RuntimeError(f"upload of {path.name} failed: HTTP {status}: {body_s}") def build_one(pkg_src: Path, cfg: Config) -> List[Path]: name = pkg_src.name build_dir = cfg.workdir / name print(f"\n=== {name} ===") reset_build_dir(build_dir, cfg.dry_run) prepare_build_dir(pkg_src, build_dir, cfg) if not cfg.dry_run and not cfg.force and already_built(build_dir, cfg): info = srcinfo(build_dir) print(f" {name} {info['fullver']} already on registry; skipping build") return [] run(cfg.makepkg_cmd, cwd=build_dir, dry_run=cfg.dry_run) if cfg.dry_run: return [] artifacts = find_artifacts(build_dir, want_sig=cfg.sign) pkgs = [a for a in artifacts if not a.name.endswith(".sig")] if not pkgs: raise RuntimeError(f"{name}: no *.pkg.tar.* produced") print(f" built: {', '.join(a.name for a in pkgs)}") return artifacts def build_config(args: argparse.Namespace) -> Config: makepkg_cmd = ["makepkg", "-s", "--noconfirm", "--needed"] if args.cleanbuild: makepkg_cmd.append("--cleanbuild") if args.skip_pgp: makepkg_cmd.append("--skippgpcheck") if args.sign: makepkg_cmd.append("--sign") return Config( repo_root=args.repo_root.resolve(), workdir=args.workdir.resolve(), makepkg_cmd=makepkg_cmd, sign=args.sign, keep_going=args.keep_going, dry_run=args.dry_run, publish=not args.no_publish, replace=args.replace, force=args.force, registry_url=args.registry_url, owner=args.owner, arch_repo=args.arch_repo, auth_user=args.auth_user or args.owner, token=args.token, ignore=load_ignore(args.repo_root.resolve()), ) def main(argv: List[str]) -> int: ap = argparse.ArgumentParser(description="Build Arch packages and publish to Gitea") ap.add_argument("--repo-root", type=Path, default=Path.cwd()) ap.add_argument("--workdir", type=Path, default=Path.cwd() / ".build") ap.add_argument("--only", nargs="*", default=[]) ap.add_argument("--list-json", action="store_true") ap.add_argument("--cleanbuild", action="store_true") ap.add_argument("--skip-pgp", action="store_true", default=env("AUTOPKG_SKIP_PGP", "") not in ("", "0", "false")) ap.add_argument("--sign", action="store_true") ap.add_argument("--keep-going", action="store_true") ap.add_argument("--dry-run", action="store_true") ap.add_argument("--no-publish", action="store_true") ap.add_argument("--replace", action="store_true", default=env("AUTOPKG_REPLACE", "") not in ("", "0", "false")) ap.add_argument("--force", action="store_true", default=env("AUTOPKG_FORCE", "") not in ("", "0", "false")) ap.add_argument("--registry-url", default=env("AUTOPKG_REGISTRY_URL", env("GITHUB_SERVER_URL"))) ap.add_argument("--owner", default=env("AUTOPKG_OWNER", env("GITHUB_REPOSITORY_OWNER"))) ap.add_argument("--arch-repo", default=env("AUTOPKG_REPO", "arch")) ap.add_argument("--auth-user", default=env("AUTOPKG_USER", env("GITHUB_ACTOR"))) ap.add_argument("--token", default=env("AUTOPKG_TOKEN", env("PACKAGE_TOKEN", env("GITHUB_TOKEN")))) args = ap.parse_args(argv) cfg = build_config(args) if args.list_json: print(json.dumps([p.name for p in discover_packages(cfg, args.only)])) return 0 pkgs = discover_packages(cfg, args.only) if not pkgs: print("No package directories found.", file=sys.stderr) return 2 print(f"Discovered {len(pkgs)} package(s): {', '.join(p.name for p in pkgs)}") if cfg.publish and not cfg.dry_run: missing = [n for n, v in (("registry-url", cfg.registry_url), ("owner", cfg.owner), ("token", cfg.token)) if not v] if missing: print(f"Cannot publish: missing {', '.join(missing)}", file=sys.stderr) return 2 failures = [] for pkg_src in pkgs: try: artifacts = build_one(pkg_src, cfg) if cfg.publish and artifacts and not cfg.dry_run: for art in artifacts: publish_file(art, cfg) elif cfg.publish and cfg.dry_run: print(f" (dry-run) would publish {pkg_src.name}") except (subprocess.CalledProcessError, RuntimeError, OSError) as e: print(f"✗ {pkg_src.name}: {e}", file=sys.stderr) failures.append(pkg_src.name) if not cfg.keep_going: break if failures: print(f"\nFailed: {', '.join(failures)}", file=sys.stderr) return 1 print("\nAll done.") return 0 if __name__ == "__main__": raise SystemExit(main(sys.argv[1:]))