433 lines
15 KiB
Python
433 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
autopackage.py
|
|
|
|
This is a rewrite/port of my original autopackage python script, that i used
|
|
to just keep to myself but... now im just trusting enough of my personal supply chain
|
|
that i think im comfy sharing package builds... at least for friends!
|
|
|
|
I mostly intend for CI to run this script but, if you want to make tweaks or experiment locally
|
|
thats super fine too!
|
|
|
|
Examples:
|
|
./autopackage.py --dry-run
|
|
./autopackage.py --only openrgb-git paru
|
|
./autopackage.py --no-publish # build everything, skip upload
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import os
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import List, Optional
|
|
|
|
AUR_BASE = "https://aur.archlinux.org"
|
|
|
|
# Top-level directories that are never packages
|
|
DEFAULT_IGNORE = {
|
|
".git",
|
|
".gitea",
|
|
".github",
|
|
".build",
|
|
"scripts",
|
|
"dist",
|
|
"__pycache__",
|
|
}
|
|
|
|
PKG_GLOBS = ("*.pkg.tar.zst", "*.pkg.tar.xz", "*.pkg.tar.gz", "*.pkg.tar.bz2")
|
|
|
|
|
|
@dataclass
|
|
class Config:
|
|
repo_root: Path
|
|
workdir: Path
|
|
makepkg_cmd: List[str]
|
|
sign: bool
|
|
keep_going: bool
|
|
dry_run: bool
|
|
publish: bool
|
|
replace: bool
|
|
# Gitea Arch registry settings
|
|
registry_url: str
|
|
owner: str
|
|
arch_repo: str
|
|
auth_user: str
|
|
token: str
|
|
ignore: set = field(default_factory=set)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Shell helpers
|
|
# --------------------------------------------------------------------------- #
|
|
def run(cmd: List[str], cwd: Optional[Path] = None, dry_run: bool = False,
|
|
check: bool = True) -> subprocess.CompletedProcess:
|
|
pretty = " ".join(shlex.quote(c) for c in cmd)
|
|
if cwd:
|
|
pretty = f"(cd {cwd}) {pretty}"
|
|
print(f"+ {pretty}", flush=True)
|
|
if dry_run:
|
|
return subprocess.CompletedProcess(cmd, 0)
|
|
return subprocess.run(cmd, cwd=str(cwd) if cwd else None, check=check)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Discovery / classification
|
|
# --------------------------------------------------------------------------- #
|
|
def load_ignore(repo_root: Path) -> set:
|
|
ignore = set(DEFAULT_IGNORE)
|
|
ignore_file = repo_root / ".autopackageignore"
|
|
if ignore_file.is_file():
|
|
for line in ignore_file.read_text(encoding="utf-8").splitlines():
|
|
line = line.strip()
|
|
if line and not line.startswith("#"):
|
|
ignore.add(line)
|
|
return ignore
|
|
|
|
|
|
def discover_packages(cfg: Config, only: List[str]) -> List[Path]:
|
|
pkgs: List[Path] = []
|
|
for entry in sorted(cfg.repo_root.iterdir()):
|
|
if not entry.is_dir():
|
|
continue
|
|
name = entry.name
|
|
if name.startswith(".") or name in cfg.ignore:
|
|
continue
|
|
if only and name not in only:
|
|
continue
|
|
pkgs.append(entry)
|
|
return pkgs
|
|
|
|
|
|
def is_custom(pkg_src: Path) -> bool:
|
|
return (pkg_src / "PKGBUILD").is_file()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Build preparation
|
|
# --------------------------------------------------------------------------- #
|
|
def reset_build_dir(build_dir: Path, dry_run: bool) -> None:
|
|
if dry_run:
|
|
print(f"+ rm -rf {build_dir}")
|
|
return
|
|
if build_dir.exists():
|
|
shutil.rmtree(build_dir)
|
|
build_dir.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
def prepare_custom(pkg_src: Path, build_dir: Path, dry_run: bool) -> None:
|
|
print(f" custom package -> copying sources into {build_dir}")
|
|
if dry_run:
|
|
print(f"+ cp -r {pkg_src} {build_dir}")
|
|
return
|
|
shutil.copytree(pkg_src, build_dir, ignore=shutil.ignore_patterns(".git"))
|
|
|
|
|
|
def prepare_aur(pkg_src: Path, build_dir: Path, cfg: Config) -> None:
|
|
name = pkg_src.name
|
|
aur_url = f"{AUR_BASE}/{name}.git"
|
|
print(f" AUR package -> cloning {aur_url}")
|
|
run(["git", "clone", "--depth", "1", aur_url, str(build_dir)], dry_run=cfg.dry_run)
|
|
apply_overrides(pkg_src, build_dir, cfg)
|
|
|
|
|
|
def apply_overrides(pkg_src: Path, build_dir: Path, cfg: Config) -> None:
|
|
patches = sorted(
|
|
p for p in pkg_src.iterdir()
|
|
if p.is_file() and p.suffix in (".patch", ".diff")
|
|
)
|
|
overrides = sorted(
|
|
p for p in pkg_src.iterdir()
|
|
if p.is_file()
|
|
and p.suffix not in (".patch", ".diff")
|
|
and p.name != ".gitkeep"
|
|
)
|
|
|
|
for ov in overrides:
|
|
dest = build_dir / ov.name
|
|
print(f" override: {ov.name}")
|
|
if cfg.dry_run:
|
|
print(f"+ cp {ov} {dest}")
|
|
continue
|
|
shutil.copy2(ov, dest)
|
|
|
|
for patch in patches:
|
|
print(f" patch: {patch.name}")
|
|
apply_patch(patch, build_dir, cfg.dry_run)
|
|
|
|
|
|
def apply_patch(patch: Path, build_dir: Path, dry_run: bool) -> None:
|
|
# Use `git apply` first if theres a patch that i have in there
|
|
git_apply = run(
|
|
["git", "apply", "--whitespace=nowarn", str(patch)],
|
|
cwd=build_dir, dry_run=dry_run, check=False,
|
|
)
|
|
if git_apply.returncode == 0:
|
|
return
|
|
if dry_run:
|
|
return
|
|
print(f" (git apply failed, trying patch -p1) {patch.name}")
|
|
with patch.open("rb") as fh:
|
|
result = subprocess.run(
|
|
["patch", "-Np1", "--forward"],
|
|
cwd=str(build_dir), stdin=fh,
|
|
)
|
|
if result.returncode != 0:
|
|
raise RuntimeError(f"failed to apply patch {patch.name}")
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Build
|
|
# --------------------------------------------------------------------------- #
|
|
def find_artifacts(build_dir: Path, want_sig: bool) -> List[Path]:
|
|
found: List[Path] = []
|
|
for pat in PKG_GLOBS:
|
|
found.extend(build_dir.glob(pat))
|
|
pkgs = [p for p in found if p.is_file() and not p.name.endswith(".sig")]
|
|
pkgs.sort(key=lambda p: p.stat().st_mtime, reverse=True)
|
|
artifacts: List[Path] = []
|
|
for p in pkgs:
|
|
artifacts.append(p)
|
|
if want_sig:
|
|
sig = p.with_name(p.name + ".sig")
|
|
if sig.is_file():
|
|
artifacts.append(sig)
|
|
return artifacts
|
|
|
|
|
|
def build_one(pkg_src: Path, cfg: Config) -> List[Path]:
|
|
name = pkg_src.name
|
|
build_dir = cfg.workdir / name
|
|
print(f"\n=== {name} ===")
|
|
|
|
reset_build_dir(build_dir, cfg.dry_run)
|
|
if is_custom(pkg_src):
|
|
prepare_custom(pkg_src, build_dir, cfg.dry_run)
|
|
else:
|
|
prepare_aur(pkg_src, build_dir, cfg)
|
|
|
|
run(cfg.makepkg_cmd, cwd=build_dir, dry_run=cfg.dry_run)
|
|
|
|
if cfg.dry_run:
|
|
return []
|
|
|
|
artifacts = find_artifacts(build_dir, want_sig=cfg.sign)
|
|
pkg_files = [a for a in artifacts if not a.name.endswith(".sig")]
|
|
if not pkg_files:
|
|
raise RuntimeError(f"{name}: build finished but no *.pkg.tar.* produced")
|
|
print(f" built: {', '.join(a.name for a in pkg_files)}")
|
|
return artifacts
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Publish to Gitea Arch registry
|
|
# --------------------------------------------------------------------------- #
|
|
def parse_pkg_filename(filename: str) -> Optional[tuple]:
|
|
"""Return (name, version, arch) from <name>-<ver>-<rel>-<arch>.pkg.tar.zst."""
|
|
base = filename
|
|
for suffix in (".pkg.tar.zst", ".pkg.tar.xz", ".pkg.tar.gz", ".pkg.tar.bz2"):
|
|
if base.endswith(suffix):
|
|
base = base[: -len(suffix)]
|
|
break
|
|
else:
|
|
return None
|
|
parts = base.rsplit("-", 3)
|
|
if len(parts) != 4:
|
|
return None
|
|
name, ver, rel, arch = parts
|
|
return name, f"{ver}-{rel}", arch
|
|
|
|
|
|
def registry_base(cfg: Config) -> str:
|
|
return f"{cfg.registry_url.rstrip('/')}/api/packages/{cfg.owner}/arch/{cfg.arch_repo}"
|
|
|
|
|
|
def auth_header(cfg: Config) -> str:
|
|
raw = f"{cfg.auth_user}:{cfg.token}".encode("utf-8")
|
|
return "Basic " + base64.b64encode(raw).decode("ascii")
|
|
|
|
|
|
def http_request(url: str, method: str, cfg: Config,
|
|
data: Optional[bytes] = None) -> tuple:
|
|
req = urllib.request.Request(url, data=data, method=method)
|
|
req.add_header("Authorization", auth_header(cfg))
|
|
if data is not None:
|
|
req.add_header("Content-Type", "application/octet-stream")
|
|
try:
|
|
with urllib.request.urlopen(req) as resp:
|
|
return resp.status, resp.read().decode("utf-8", "replace")
|
|
except urllib.error.HTTPError as e:
|
|
return e.code, e.read().decode("utf-8", "replace")
|
|
|
|
|
|
def delete_version(filename: str, cfg: Config) -> None:
|
|
meta = parse_pkg_filename(filename)
|
|
if not meta:
|
|
return
|
|
name, version, arch = meta
|
|
url = f"{registry_base(cfg)}/{name}/{version}/{arch}"
|
|
status, body = http_request(url, "DELETE", cfg)
|
|
print(f" delete {name} {version} {arch} -> {status}")
|
|
|
|
|
|
def publish_file(path: Path, cfg: Config) -> None:
|
|
url = registry_base(cfg)
|
|
if cfg.dry_run:
|
|
print(f"+ PUT {url} ({path.name})")
|
|
return
|
|
data = path.read_bytes()
|
|
status, body = http_request(url, "PUT", cfg, data=data)
|
|
if status in (200, 201):
|
|
print(f" uploaded {path.name} -> {status}")
|
|
return
|
|
if status == 409 and cfg.replace:
|
|
print(f" {path.name} exists; replacing")
|
|
delete_version(path.name, cfg)
|
|
status, body = http_request(url, "PUT", cfg, data=data)
|
|
if status in (200, 201):
|
|
print(f" uploaded {path.name} -> {status}")
|
|
return
|
|
if status == 409:
|
|
print(f" {path.name} already published (409); skipping "
|
|
f"(use --replace to overwrite)")
|
|
return
|
|
raise RuntimeError(
|
|
f"upload of {path.name} failed: HTTP {status}: {body.strip()[:500]}"
|
|
)
|
|
|
|
|
|
def publish(artifacts: List[Path], cfg: Config) -> None:
|
|
for art in artifacts:
|
|
publish_file(art, cfg)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# CLI
|
|
# --------------------------------------------------------------------------- #
|
|
def env(name: str, default: str = "") -> str:
|
|
return os.environ.get(name, default)
|
|
|
|
|
|
def build_config(args: argparse.Namespace) -> Config:
|
|
repo_root = args.repo_root.resolve()
|
|
|
|
makepkg_cmd = ["makepkg", "-s", "--noconfirm", "--needed"]
|
|
if args.cleanbuild:
|
|
makepkg_cmd.append("--cleanbuild")
|
|
if args.sign:
|
|
makepkg_cmd.append("--sign")
|
|
|
|
cfg = Config(
|
|
repo_root=repo_root,
|
|
workdir=args.workdir.resolve(),
|
|
makepkg_cmd=makepkg_cmd,
|
|
sign=args.sign,
|
|
keep_going=args.keep_going,
|
|
dry_run=args.dry_run,
|
|
publish=not args.no_publish,
|
|
replace=args.replace,
|
|
registry_url=args.registry_url,
|
|
owner=args.owner,
|
|
arch_repo=args.arch_repo,
|
|
auth_user=args.auth_user or args.owner,
|
|
token=args.token,
|
|
ignore=load_ignore(repo_root),
|
|
)
|
|
return cfg
|
|
|
|
|
|
def main(argv: List[str]) -> int:
|
|
ap = argparse.ArgumentParser(
|
|
description="Nightly directory-driven Arch package builder for Gitea CI",
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
)
|
|
ap.add_argument("--repo-root", type=Path, default=Path.cwd(),
|
|
help="Repo root to scan for package directories")
|
|
ap.add_argument("--workdir", type=Path, default=Path.cwd() / ".build",
|
|
help="Scratch directory for clones/builds")
|
|
ap.add_argument("--only", nargs="*", default=[],
|
|
help="Only build these package directory names")
|
|
ap.add_argument("--cleanbuild", action="store_true",
|
|
help="Pass --cleanbuild to makepkg")
|
|
ap.add_argument("--sign", action="store_true",
|
|
help="Sign packages (makepkg --sign) and upload .sig files")
|
|
ap.add_argument("--keep-going", action="store_true",
|
|
help="Continue with remaining packages if one fails")
|
|
ap.add_argument("--dry-run", action="store_true",
|
|
help="Print actions without executing them")
|
|
ap.add_argument("--no-publish", action="store_true",
|
|
help="Build only; do not upload to the registry")
|
|
ap.add_argument("--replace", action="store_true",
|
|
default=env("AUTOPKG_REPLACE", "") not in ("", "0", "false"),
|
|
help="Delete + re-upload when a file already exists (409)")
|
|
|
|
# Gitea Arch registry (CLI overrides env; env defaults suit Gitea Actions)
|
|
ap.add_argument("--registry-url", default=env("AUTOPKG_REGISTRY_URL",
|
|
env("GITHUB_SERVER_URL")),
|
|
help="Gitea base URL, e.g. https://git.example.com")
|
|
ap.add_argument("--owner", default=env("AUTOPKG_OWNER",
|
|
env("GITHUB_REPOSITORY_OWNER")),
|
|
help="Package registry owner (user or org)")
|
|
ap.add_argument("--arch-repo", default=env("AUTOPKG_REPO", "arch"),
|
|
help="Registry repository/group name (the [repo] in pacman.conf)")
|
|
ap.add_argument("--auth-user", default=env("AUTOPKG_USER",
|
|
env("GITHUB_ACTOR")),
|
|
help="Username for HTTP basic auth (defaults to owner)")
|
|
ap.add_argument("--token", default=env("AUTOPKG_TOKEN",
|
|
env("PACKAGE_TOKEN", env("GITHUB_TOKEN"))),
|
|
help="Gitea token with write:package scope")
|
|
|
|
args = ap.parse_args(argv)
|
|
cfg = build_config(args)
|
|
|
|
pkgs = discover_packages(cfg, args.only)
|
|
if not pkgs:
|
|
print("No package directories found.", file=sys.stderr)
|
|
return 2
|
|
|
|
print(f"Discovered {len(pkgs)} package(s): "
|
|
f"{', '.join(p.name for p in pkgs)}")
|
|
|
|
if cfg.publish and not cfg.dry_run:
|
|
missing = [n for n, v in (("registry-url", cfg.registry_url),
|
|
("owner", cfg.owner),
|
|
("token", cfg.token)) if not v]
|
|
if missing:
|
|
print(f"Cannot publish: missing {', '.join(missing)}. "
|
|
f"Set the matching env/secret or pass --no-publish.",
|
|
file=sys.stderr)
|
|
return 2
|
|
|
|
failures: List[str] = []
|
|
for pkg_src in pkgs:
|
|
try:
|
|
artifacts = build_one(pkg_src, cfg)
|
|
if cfg.publish and not cfg.dry_run:
|
|
publish(artifacts, cfg)
|
|
elif cfg.publish and cfg.dry_run:
|
|
print(f" (dry-run) would publish artifacts for {pkg_src.name}")
|
|
except (subprocess.CalledProcessError, RuntimeError, OSError) as e:
|
|
print(f"✗ {pkg_src.name}: {e}", file=sys.stderr)
|
|
failures.append(pkg_src.name)
|
|
if not cfg.keep_going:
|
|
break
|
|
|
|
if failures:
|
|
print(f"\nFailed: {', '.join(failures)}", file=sys.stderr)
|
|
return 1
|
|
|
|
print("\nAll done.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main(sys.argv[1:]))
|