#!/usr/bin/env python3 """ autopackage.py This is a rewrite/port of my original autopackage python script, that i used to just keep to myself but... now im just trusting enough of my personal supply chain that i think im comfy sharing package builds... at least for friends! I mostly intend for CI to run this script but, if you want to make tweaks or experiment locally thats super fine too! Examples: ./autopackage.py --dry-run ./autopackage.py --only openrgb-git paru ./autopackage.py --no-publish # build everything, skip upload """ from __future__ import annotations import argparse import base64 import json import os import shlex import shutil import subprocess import sys import time import urllib.error import urllib.request from dataclasses import dataclass, field from pathlib import Path from typing import List, Optional AUR_BASE = "https://aur.archlinux.org" # Top-level directories that are never packages DEFAULT_IGNORE = { ".git", ".gitea", ".github", ".build", "scripts", "dist", "__pycache__", } PKG_GLOBS = ("*.pkg.tar.zst", "*.pkg.tar.xz", "*.pkg.tar.gz", "*.pkg.tar.bz2") @dataclass class Config: repo_root: Path workdir: Path makepkg_cmd: List[str] sign: bool keep_going: bool dry_run: bool publish: bool replace: bool # Gitea Arch registry settings registry_url: str owner: str arch_repo: str auth_user: str token: str ignore: set = field(default_factory=set) # --------------------------------------------------------------------------- # # Shell helpers # --------------------------------------------------------------------------- # def run(cmd: List[str], cwd: Optional[Path] = None, dry_run: bool = False, check: bool = True) -> subprocess.CompletedProcess: pretty = " ".join(shlex.quote(c) for c in cmd) if cwd: pretty = f"(cd {cwd}) {pretty}" print(f"+ {pretty}", flush=True) if dry_run: return subprocess.CompletedProcess(cmd, 0) return subprocess.run(cmd, cwd=str(cwd) if cwd else None, check=check) # --------------------------------------------------------------------------- # # Discovery / classification # --------------------------------------------------------------------------- # def load_ignore(repo_root: Path) -> set: ignore = set(DEFAULT_IGNORE) ignore_file = repo_root / ".autopackageignore" if ignore_file.is_file(): for line in ignore_file.read_text(encoding="utf-8").splitlines(): line = line.strip() if line and not line.startswith("#"): ignore.add(line) return ignore def discover_packages(cfg: Config, only: List[str]) -> List[Path]: pkgs: List[Path] = [] for entry in sorted(cfg.repo_root.iterdir()): if not entry.is_dir(): continue name = entry.name if name.startswith(".") or name in cfg.ignore: continue if only and name not in only: continue pkgs.append(entry) return pkgs def is_custom(pkg_src: Path) -> bool: return (pkg_src / "PKGBUILD").is_file() # --------------------------------------------------------------------------- # # Build preparation # --------------------------------------------------------------------------- # def reset_build_dir(build_dir: Path, dry_run: bool) -> None: if dry_run: print(f"+ rm -rf {build_dir}") return if build_dir.exists(): shutil.rmtree(build_dir) build_dir.parent.mkdir(parents=True, exist_ok=True) def prepare_custom(pkg_src: Path, build_dir: Path, dry_run: bool) -> None: print(f" custom package -> copying sources into {build_dir}") if dry_run: print(f"+ cp -r {pkg_src} {build_dir}") return shutil.copytree(pkg_src, build_dir, ignore=shutil.ignore_patterns(".git")) def prepare_aur(pkg_src: Path, build_dir: Path, cfg: Config) -> None: name = pkg_src.name aur_url = f"{AUR_BASE}/{name}.git" print(f" AUR package -> cloning {aur_url}") run(["git", "clone", "--depth", "1", aur_url, str(build_dir)], dry_run=cfg.dry_run) apply_overrides(pkg_src, build_dir, cfg) def apply_overrides(pkg_src: Path, build_dir: Path, cfg: Config) -> None: patches = sorted( p for p in pkg_src.iterdir() if p.is_file() and p.suffix in (".patch", ".diff") ) overrides = sorted( p for p in pkg_src.iterdir() if p.is_file() and p.suffix not in (".patch", ".diff") and p.name != ".gitkeep" ) for ov in overrides: dest = build_dir / ov.name print(f" override: {ov.name}") if cfg.dry_run: print(f"+ cp {ov} {dest}") continue shutil.copy2(ov, dest) for patch in patches: print(f" patch: {patch.name}") apply_patch(patch, build_dir, cfg.dry_run) def apply_patch(patch: Path, build_dir: Path, dry_run: bool) -> None: # Use `git apply` first if theres a patch that i have in there git_apply = run( ["git", "apply", "--whitespace=nowarn", str(patch)], cwd=build_dir, dry_run=dry_run, check=False, ) if git_apply.returncode == 0: return if dry_run: return print(f" (git apply failed, trying patch -p1) {patch.name}") with patch.open("rb") as fh: result = subprocess.run( ["patch", "-Np1", "--forward"], cwd=str(build_dir), stdin=fh, ) if result.returncode != 0: raise RuntimeError(f"failed to apply patch {patch.name}") # --------------------------------------------------------------------------- # # Build # --------------------------------------------------------------------------- # def find_artifacts(build_dir: Path, want_sig: bool) -> List[Path]: found: List[Path] = [] for pat in PKG_GLOBS: found.extend(build_dir.glob(pat)) pkgs = [p for p in found if p.is_file() and not p.name.endswith(".sig")] pkgs.sort(key=lambda p: p.stat().st_mtime, reverse=True) artifacts: List[Path] = [] for p in pkgs: artifacts.append(p) if want_sig: sig = p.with_name(p.name + ".sig") if sig.is_file(): artifacts.append(sig) return artifacts def build_one(pkg_src: Path, cfg: Config) -> List[Path]: name = pkg_src.name build_dir = cfg.workdir / name print(f"\n=== {name} ===") reset_build_dir(build_dir, cfg.dry_run) if is_custom(pkg_src): prepare_custom(pkg_src, build_dir, cfg.dry_run) else: prepare_aur(pkg_src, build_dir, cfg) run(cfg.makepkg_cmd, cwd=build_dir, dry_run=cfg.dry_run) if cfg.dry_run: return [] artifacts = find_artifacts(build_dir, want_sig=cfg.sign) pkg_files = [a for a in artifacts if not a.name.endswith(".sig")] if not pkg_files: raise RuntimeError(f"{name}: build finished but no *.pkg.tar.* produced") print(f" built: {', '.join(a.name for a in pkg_files)}") return artifacts # --------------------------------------------------------------------------- # # Publish to Gitea Arch registry # --------------------------------------------------------------------------- # def parse_pkg_filename(filename: str) -> Optional[tuple]: """Return (name, version, arch) from ---.pkg.tar.zst.""" base = filename for suffix in (".pkg.tar.zst", ".pkg.tar.xz", ".pkg.tar.gz", ".pkg.tar.bz2"): if base.endswith(suffix): base = base[: -len(suffix)] break else: return None parts = base.rsplit("-", 3) if len(parts) != 4: return None name, ver, rel, arch = parts return name, f"{ver}-{rel}", arch def registry_base(cfg: Config) -> str: return f"{cfg.registry_url.rstrip('/')}/api/packages/{cfg.owner}/arch/{cfg.arch_repo}" def auth_header(cfg: Config) -> str: raw = f"{cfg.auth_user}:{cfg.token}".encode("utf-8") return "Basic " + base64.b64encode(raw).decode("ascii") def http_request(url: str, method: str, cfg: Config, data: Optional[bytes] = None, retries: int = 4) -> tuple: last_err: Optional[Exception] = None for attempt in range(1, retries + 1): req = urllib.request.Request(url, data=data, method=method) req.add_header("Authorization", auth_header(cfg)) if data is not None: req.add_header("Content-Type", "application/octet-stream") try: with urllib.request.urlopen(req, timeout=300) as resp: return resp.status, resp.read().decode("utf-8", "replace") except urllib.error.HTTPError as e: body = e.read().decode("utf-8", "replace") # Retry transient server errors; return everything else as-is. if e.code in (500, 502, 503, 504) and attempt < retries: last_err = e else: return e.code, body except (urllib.error.URLError, ConnectionError, TimeoutError, OSError) as e: # Network-level failures (e.g. connection reset by peer) -> retry. last_err = e if attempt >= retries: raise backoff = min(2 ** attempt, 30) print(f" {method} {url} failed ({last_err}); " f"retry {attempt}/{retries - 1} in {backoff}s") time.sleep(backoff) raise RuntimeError(f"{method} {url} failed after {retries} attempts: {last_err}") def delete_version(filename: str, cfg: Config) -> None: meta = parse_pkg_filename(filename) if not meta: return name, version, arch = meta url = f"{registry_base(cfg)}/{name}/{version}/{arch}" status, body = http_request(url, "DELETE", cfg) print(f" delete {name} {version} {arch} -> {status}") def publish_file(path: Path, cfg: Config) -> None: url = registry_base(cfg) if cfg.dry_run: print(f"+ PUT {url} ({path.name})") return data = path.read_bytes() status, body = http_request(url, "PUT", cfg, data=data) if status in (200, 201): print(f" uploaded {path.name} -> {status}") return if status == 409 and cfg.replace: print(f" {path.name} exists; replacing") delete_version(path.name, cfg) status, body = http_request(url, "PUT", cfg, data=data) if status in (200, 201): print(f" uploaded {path.name} -> {status}") return if status == 409: print(f" {path.name} already published (409); skipping " f"(use --replace to overwrite)") return if status in (401, 403): raise RuntimeError( f"upload of {path.name} failed: HTTP {status}: " f"{body.strip()[:200]}\n" f" -> The registry rejected the credentials for owner " f"'{cfg.owner}'.\n" f" Gitea's automatic Actions token (github.token / " f"GITEA_TOKEN) cannot write packages.\n" f" Use a Personal Access Token with the 'write:package' scope " f"(set PACKAGE_TOKEN / --token)." ) raise RuntimeError( f"upload of {path.name} failed: HTTP {status}: {body.strip()[:500]}" ) def publish(artifacts: List[Path], cfg: Config) -> None: for art in artifacts: publish_file(art, cfg) # --------------------------------------------------------------------------- # # CLI # --------------------------------------------------------------------------- # def env(name: str, default: str = "") -> str: return os.environ.get(name, default) def build_config(args: argparse.Namespace) -> Config: repo_root = args.repo_root.resolve() makepkg_cmd = ["makepkg", "-s", "--noconfirm", "--needed"] if args.cleanbuild: makepkg_cmd.append("--cleanbuild") if args.skip_pgp: makepkg_cmd.append("--skippgpcheck") if args.sign: makepkg_cmd.append("--sign") cfg = Config( repo_root=repo_root, workdir=args.workdir.resolve(), makepkg_cmd=makepkg_cmd, sign=args.sign, keep_going=args.keep_going, dry_run=args.dry_run, publish=not args.no_publish, replace=args.replace, registry_url=args.registry_url, owner=args.owner, arch_repo=args.arch_repo, auth_user=args.auth_user or args.owner, token=args.token, ignore=load_ignore(repo_root), ) return cfg def main(argv: List[str]) -> int: ap = argparse.ArgumentParser( description="Nightly directory-driven Arch package builder for Gitea CI", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) ap.add_argument("--repo-root", type=Path, default=Path.cwd(), help="Repo root to scan for package directories") ap.add_argument("--workdir", type=Path, default=Path.cwd() / ".build", help="Scratch directory for clones/builds") ap.add_argument("--only", nargs="*", default=[], help="Only build these package directory names") ap.add_argument("--list-json", action="store_true", help="Print discovered package names as a JSON array and exit " "(used to build the CI matrix)") ap.add_argument("--cleanbuild", action="store_true", help="Pass --cleanbuild to makepkg") ap.add_argument("--skip-pgp", action="store_true", default=env("AUTOPKG_SKIP_PGP", "") not in ("", "0", "false"), help="Skip source PGP signature checks (sha256sums still " "verified). Useful when upstream signing keys aren't in " "the build keyring.") ap.add_argument("--sign", action="store_true", help="Sign packages (makepkg --sign) and upload .sig files") ap.add_argument("--keep-going", action="store_true", help="Continue with remaining packages if one fails") ap.add_argument("--dry-run", action="store_true", help="Print actions without executing them") ap.add_argument("--no-publish", action="store_true", help="Build only; do not upload to the registry") ap.add_argument("--replace", action="store_true", default=env("AUTOPKG_REPLACE", "") not in ("", "0", "false"), help="Delete + re-upload when a file already exists (409)") # Gitea Arch registry (CLI overrides env; env defaults suit Gitea Actions) ap.add_argument("--registry-url", default=env("AUTOPKG_REGISTRY_URL", env("GITHUB_SERVER_URL")), help="Gitea base URL, e.g. https://git.example.com") ap.add_argument("--owner", default=env("AUTOPKG_OWNER", env("GITHUB_REPOSITORY_OWNER")), help="Package registry owner (user or org)") ap.add_argument("--arch-repo", default=env("AUTOPKG_REPO", "arch"), help="Registry repository/group name (the [repo] in pacman.conf)") ap.add_argument("--auth-user", default=env("AUTOPKG_USER", env("GITHUB_ACTOR")), help="Username for HTTP basic auth (defaults to owner)") ap.add_argument("--token", default=env("AUTOPKG_TOKEN", env("PACKAGE_TOKEN", env("GITHUB_TOKEN"))), help="Gitea token with write:package scope") args = ap.parse_args(argv) cfg = build_config(args) if args.list_json: pkgs = discover_packages(cfg, args.only) print(json.dumps([p.name for p in pkgs])) return 0 pkgs = discover_packages(cfg, args.only) if not pkgs: print("No package directories found.", file=sys.stderr) return 2 print(f"Discovered {len(pkgs)} package(s): " f"{', '.join(p.name for p in pkgs)}") if cfg.publish and not cfg.dry_run: missing = [n for n, v in (("registry-url", cfg.registry_url), ("owner", cfg.owner), ("token", cfg.token)) if not v] if missing: print(f"Cannot publish: missing {', '.join(missing)}. " f"Set the matching env/secret or pass --no-publish.", file=sys.stderr) return 2 failures: List[str] = [] for pkg_src in pkgs: try: artifacts = build_one(pkg_src, cfg) if cfg.publish and not cfg.dry_run: publish(artifacts, cfg) elif cfg.publish and cfg.dry_run: print(f" (dry-run) would publish artifacts for {pkg_src.name}") except (subprocess.CalledProcessError, RuntimeError, OSError) as e: print(f"✗ {pkg_src.name}: {e}", file=sys.stderr) failures.append(pkg_src.name) if not cfg.keep_going: break if failures: print(f"\nFailed: {', '.join(failures)}", file=sys.stderr) return 1 print("\nAll done.") return 0 if __name__ == "__main__": raise SystemExit(main(sys.argv[1:]))