d5eb2f2045
Some checks failed
nightly-build / build (bacula) (push) Successful in 1m19s
nightly-build / build (fchat-horizon-appimage) (push) Successful in 47s
nightly-build / build (openrgb-git) (push) Failing after 44s
nightly-build / build (paru) (push) Successful in 55s
nightly-build / build (plezy) (push) Successful in 49s
nightly-build / build (qownnotes) (push) Successful in 52s
nightly-build / build (ventoy-bin) (push) Successful in 42s
nightly-build / build (visual-studio-code-bin) (push) Successful in 47s
520 lines
17 KiB
Python
520 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
autopackage.py
|
|
|
|
This is a rewrite/port of my original autopackage python script, that i used
|
|
to just keep to myself but... now im just trusting enough of my personal supply chain
|
|
that i think im comfy sharing package builds... at least for friends!
|
|
|
|
I mostly intend for CI to run this script but, if you want to make tweaks or experiment locally
|
|
thats super fine too!
|
|
|
|
Examples:
|
|
./autopackage.py --dry-run
|
|
./autopackage.py --only openrgb-git paru
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import List, Optional
|
|
|
|
AUR_BASE = "https://aur.archlinux.org"
|
|
DEFAULT_IGNORE = {".git", ".gitea", ".github", ".build", "scripts", "dist", "__pycache__"}
|
|
PKG_GLOBS = ("*.pkg.tar.zst", "*.pkg.tar.xz", "*.pkg.tar.gz", "*.pkg.tar.bz2")
|
|
PATCH_SUFFIXES = (".patch", ".diff")
|
|
|
|
|
|
@dataclass
|
|
class Config:
|
|
repo_root: Path
|
|
workdir: Path
|
|
makepkg_cmd: List[str]
|
|
sign: bool
|
|
keep_going: bool
|
|
dry_run: bool
|
|
publish: bool
|
|
replace: bool
|
|
force: bool
|
|
registry_url: str
|
|
owner: str
|
|
arch_repo: str
|
|
auth_user: str
|
|
token: str
|
|
ignore: set = field(default_factory=set)
|
|
|
|
|
|
def run(cmd: List[str], cwd: Optional[Path] = None, dry_run: bool = False,
|
|
check: bool = True) -> subprocess.CompletedProcess:
|
|
pretty = " ".join(shlex.quote(c) for c in cmd)
|
|
if cwd:
|
|
pretty = f"(cd {cwd}) {pretty}"
|
|
print(f"+ {pretty}", flush=True)
|
|
if dry_run:
|
|
return subprocess.CompletedProcess(cmd, 0)
|
|
return subprocess.run(cmd, cwd=str(cwd) if cwd else None, check=check)
|
|
|
|
|
|
def env(name: str, default: str = "") -> str:
|
|
return os.environ.get(name, default)
|
|
|
|
|
|
def load_ignore(repo_root: Path) -> set:
|
|
ignore = set(DEFAULT_IGNORE)
|
|
path = repo_root / ".autopackageignore"
|
|
if path.is_file():
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
line = line.strip()
|
|
if line and not line.startswith("#"):
|
|
ignore.add(line)
|
|
return ignore
|
|
|
|
|
|
def discover_packages(cfg: Config, only: List[str]) -> List[Path]:
|
|
pkgs = []
|
|
for entry in sorted(cfg.repo_root.iterdir()):
|
|
if not entry.is_dir() or entry.name.startswith(".") or entry.name in cfg.ignore:
|
|
continue
|
|
if only and entry.name not in only:
|
|
continue
|
|
pkgs.append(entry)
|
|
return pkgs
|
|
|
|
|
|
def package_files(pkg_src: Path) -> List[Path]:
|
|
return sorted(
|
|
p for p in pkg_src.iterdir()
|
|
if p.is_file() and p.name != ".gitkeep"
|
|
)
|
|
|
|
|
|
def is_custom(pkg_src: Path) -> bool:
|
|
return (pkg_src / "PKGBUILD").is_file()
|
|
|
|
|
|
def reset_build_dir(build_dir: Path, dry_run: bool) -> None:
|
|
if dry_run:
|
|
print(f"+ rm -rf {build_dir}")
|
|
return
|
|
if build_dir.exists():
|
|
shutil.rmtree(build_dir)
|
|
build_dir.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
def prepare_build_dir(pkg_src: Path, build_dir: Path, cfg: Config) -> None:
|
|
name = pkg_src.name
|
|
if is_custom(pkg_src):
|
|
print(f" custom -> copy {pkg_src}")
|
|
if cfg.dry_run:
|
|
print(f"+ cp -r {pkg_src} {build_dir}")
|
|
return
|
|
shutil.copytree(pkg_src, build_dir, ignore=shutil.ignore_patterns(".git"))
|
|
return
|
|
|
|
print(f" AUR -> clone {AUR_BASE}/{name}.git")
|
|
run(["git", "clone", "--depth", "1", f"{AUR_BASE}/{name}.git", str(build_dir)],
|
|
dry_run=cfg.dry_run)
|
|
|
|
patches: List[str] = []
|
|
source_patches: List[str] = []
|
|
for src in package_files(pkg_src):
|
|
if src.suffix in PATCH_SUFFIXES:
|
|
patches.append(src.name)
|
|
print(f" patch: {src.name}")
|
|
else:
|
|
print(f" override: {src.name}")
|
|
if cfg.dry_run:
|
|
print(f"+ cp {src} {build_dir / src.name}")
|
|
continue
|
|
shutil.copy2(src, build_dir / src.name)
|
|
|
|
if cfg.dry_run:
|
|
if patches:
|
|
print(f"+ apply packaging patches (if any), wire rest into PKGBUILD")
|
|
return
|
|
|
|
for patch in patches:
|
|
path = build_dir / patch
|
|
if apply_packaging_patch(path, build_dir):
|
|
print(f" packaging patch applied: {patch}")
|
|
else:
|
|
source_patches.append(patch)
|
|
|
|
if source_patches:
|
|
wire_source_patches(build_dir / "PKGBUILD", source_patches)
|
|
|
|
|
|
def apply_packaging_patch(patch: Path, build_dir: Path) -> bool:
|
|
"""Apply a patch to the AUR packaging tree (PKGBUILD etc.), if it fits."""
|
|
check = subprocess.run(
|
|
["git", "apply", "--check", "--whitespace=nowarn", str(patch)],
|
|
cwd=str(build_dir), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
)
|
|
if check.returncode != 0:
|
|
return False
|
|
run(["git", "apply", "--whitespace=nowarn", str(patch)], cwd=build_dir)
|
|
return True
|
|
|
|
|
|
def wire_source_patches(pkgbuild: Path, patches: List[str]) -> None:
|
|
"""Append patches to source= and apply them in prepare() after the first cd."""
|
|
text = pkgbuild.read_text(encoding="utf-8")
|
|
for patch in patches:
|
|
if patch in text:
|
|
continue
|
|
text = text.rstrip() + f'\nsource+=("{patch}")\nsha256sums+=("SKIP")\n'
|
|
m = re.search(r"(prepare\(\)\s*\{[^\n]*\n\s*cd [^\n]+\n)", text)
|
|
if not m:
|
|
raise RuntimeError(
|
|
f"{pkgbuild}: need a prepare() with a cd line to apply {patch}"
|
|
)
|
|
insert = f' patch -Np1 -i "$srcdir/{patch}"\n'
|
|
text = text[:m.end()] + insert + text[m.end():]
|
|
pkgbuild.write_text(text, encoding="utf-8")
|
|
|
|
|
|
def find_artifacts(build_dir: Path, want_sig: bool) -> List[Path]:
|
|
found: List[Path] = []
|
|
for pat in PKG_GLOBS:
|
|
found.extend(build_dir.glob(pat))
|
|
pkgs = sorted(
|
|
(p for p in found if p.is_file() and not p.name.endswith(".sig")),
|
|
key=lambda p: p.stat().st_mtime, reverse=True,
|
|
)
|
|
out: List[Path] = []
|
|
for p in pkgs:
|
|
out.append(p)
|
|
if want_sig:
|
|
sig = p.with_name(p.name + ".sig")
|
|
if sig.is_file():
|
|
out.append(sig)
|
|
return out
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Skip rebuild when this version is already on the registry
|
|
# --------------------------------------------------------------------------- #
|
|
_REGISTRY_FILES: Optional[set] = None
|
|
|
|
|
|
def registry_base(cfg: Config) -> str:
|
|
return f"{cfg.registry_url.rstrip('/')}/api/packages/{cfg.owner}/arch/{cfg.arch_repo}"
|
|
|
|
|
|
def auth_header(cfg: Config) -> str:
|
|
raw = f"{cfg.auth_user}:{cfg.token}".encode()
|
|
return "Basic " + base64.b64encode(raw).decode("ascii")
|
|
|
|
|
|
def http(url: str, method: str, cfg: Config, data: Optional[bytes] = None,
|
|
retries: int = 4) -> tuple:
|
|
last_err: Optional[Exception] = None
|
|
for attempt in range(1, retries + 1):
|
|
req = urllib.request.Request(url, data=data, method=method)
|
|
if cfg.token:
|
|
req.add_header("Authorization", auth_header(cfg))
|
|
if data is not None:
|
|
req.add_header("Content-Type", "application/octet-stream")
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=300) as resp:
|
|
return resp.status, resp.read()
|
|
except urllib.error.HTTPError as e:
|
|
body = e.read()
|
|
if e.code == 404:
|
|
return 404, body
|
|
if e.code in (500, 502, 503, 504) and attempt < retries:
|
|
last_err = e
|
|
else:
|
|
return e.code, body
|
|
except (urllib.error.URLError, ConnectionError, TimeoutError, OSError) as e:
|
|
last_err = e
|
|
if attempt >= retries:
|
|
raise
|
|
time.sleep(min(2 ** attempt, 30))
|
|
raise RuntimeError(f"{method} {url} failed: {last_err}")
|
|
|
|
|
|
def registry_filenames(cfg: Config) -> Optional[set]:
|
|
global _REGISTRY_FILES
|
|
if _REGISTRY_FILES is not None:
|
|
return _REGISTRY_FILES
|
|
if not cfg.registry_url or not cfg.owner:
|
|
return None
|
|
|
|
names: set = set()
|
|
ok = False
|
|
for arch in ("x86_64", "any"):
|
|
url = f"{registry_base(cfg)}/{arch}/{cfg.arch_repo}.db"
|
|
try:
|
|
status, data = http(url, "GET", cfg)
|
|
except (RuntimeError, OSError):
|
|
continue
|
|
if status == 404:
|
|
ok = True
|
|
continue
|
|
if status != 200 or not data:
|
|
continue
|
|
ok = True
|
|
try:
|
|
with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tf:
|
|
for m in tf.getmembers():
|
|
if not m.isfile() or not m.name.endswith("/desc"):
|
|
continue
|
|
fh = tf.extractfile(m)
|
|
if not fh:
|
|
continue
|
|
lines = fh.read().decode("utf-8", "replace").splitlines()
|
|
for i, line in enumerate(lines):
|
|
if line.strip() == "%FILENAME%" and i + 1 < len(lines):
|
|
names.add(lines[i + 1].strip())
|
|
break
|
|
except (tarfile.TarError, OSError):
|
|
continue
|
|
|
|
if not ok:
|
|
return None
|
|
_REGISTRY_FILES = names
|
|
return names
|
|
|
|
|
|
def srcinfo(build_dir: Path) -> Optional[dict]:
|
|
pkgbuild = build_dir / "PKGBUILD"
|
|
if not pkgbuild.is_file():
|
|
return None
|
|
if re.search(r"^\s*pkgver\s*\(\)", pkgbuild.read_text(errors="replace"), re.M):
|
|
return {"vcs": True}
|
|
try:
|
|
out = subprocess.run(
|
|
["makepkg", "--printsrcinfo"], cwd=build_dir,
|
|
capture_output=True, text=True, check=True,
|
|
).stdout
|
|
except (subprocess.CalledProcessError, OSError):
|
|
return None
|
|
|
|
info = {"vcs": False, "names": [], "pkgver": None, "pkgrel": None,
|
|
"epoch": None, "arch": "x86_64"}
|
|
for line in out.splitlines():
|
|
if "=" not in line:
|
|
continue
|
|
key, val = (p.strip() for p in line.split("=", 1))
|
|
if key == "pkgname":
|
|
info["names"].append(val)
|
|
elif key == "pkgver":
|
|
info["pkgver"] = val
|
|
elif key == "pkgrel":
|
|
info["pkgrel"] = val
|
|
elif key == "epoch":
|
|
info["epoch"] = val
|
|
elif key == "arch":
|
|
if val == "x86_64":
|
|
info["arch"] = "x86_64"
|
|
elif val == "any" and info["arch"] != "x86_64":
|
|
info["arch"] = "any"
|
|
if not info["names"] or not info["pkgver"] or not info["pkgrel"]:
|
|
return None
|
|
ver = info["pkgver"]
|
|
if info["epoch"]:
|
|
ver = f"{info['epoch']}:{ver}"
|
|
info["fullver"] = f"{ver}-{info['pkgrel']}"
|
|
return info
|
|
|
|
|
|
def already_built(build_dir: Path, cfg: Config) -> bool:
|
|
info = srcinfo(build_dir)
|
|
if not info or info.get("vcs"):
|
|
return False
|
|
published = registry_filenames(cfg)
|
|
if published is None:
|
|
return False
|
|
arch = info["arch"]
|
|
for name in info["names"]:
|
|
stem = f"{name}-{info['fullver']}-{arch}.pkg.tar"
|
|
if not any(f.startswith(stem) for f in published):
|
|
return False
|
|
return True
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Publish
|
|
# --------------------------------------------------------------------------- #
|
|
def parse_pkg_filename(filename: str) -> Optional[tuple]:
|
|
base = filename
|
|
for suffix in (".pkg.tar.zst", ".pkg.tar.xz", ".pkg.tar.gz", ".pkg.tar.bz2"):
|
|
if base.endswith(suffix):
|
|
base = base[: -len(suffix)]
|
|
break
|
|
else:
|
|
return None
|
|
parts = base.rsplit("-", 3)
|
|
if len(parts) != 4:
|
|
return None
|
|
name, ver, rel, arch = parts
|
|
return name, f"{ver}-{rel}", arch
|
|
|
|
|
|
def publish_file(path: Path, cfg: Config) -> None:
|
|
url = registry_base(cfg)
|
|
if cfg.dry_run:
|
|
print(f"+ PUT {url} ({path.name})")
|
|
return
|
|
data = path.read_bytes()
|
|
status, body = http(url, "PUT", cfg, data=data)
|
|
body_s = body.decode("utf-8", "replace").strip()[:500]
|
|
if status in (200, 201):
|
|
print(f" uploaded {path.name} -> {status}")
|
|
return
|
|
if status == 409 and cfg.replace:
|
|
meta = parse_pkg_filename(path.name)
|
|
if meta:
|
|
n, ver, arch = meta
|
|
http(f"{registry_base(cfg)}/{n}/{ver}/{arch}", "DELETE", cfg)
|
|
status, body = http(url, "PUT", cfg, data=data)
|
|
body_s = body.decode("utf-8", "replace").strip()[:500]
|
|
if status in (200, 201):
|
|
print(f" uploaded {path.name} -> {status}")
|
|
return
|
|
if status == 409:
|
|
print(f" {path.name} already published (409); skipping")
|
|
return
|
|
if status in (401, 403):
|
|
raise RuntimeError(
|
|
f"upload of {path.name} failed: HTTP {status}: {body_s}\n"
|
|
f" -> need a PAT with write:package (PACKAGE_TOKEN), not github.token"
|
|
)
|
|
raise RuntimeError(f"upload of {path.name} failed: HTTP {status}: {body_s}")
|
|
|
|
|
|
def build_one(pkg_src: Path, cfg: Config) -> List[Path]:
|
|
name = pkg_src.name
|
|
build_dir = cfg.workdir / name
|
|
print(f"\n=== {name} ===")
|
|
|
|
reset_build_dir(build_dir, cfg.dry_run)
|
|
prepare_build_dir(pkg_src, build_dir, cfg)
|
|
|
|
if not cfg.dry_run and not cfg.force and already_built(build_dir, cfg):
|
|
info = srcinfo(build_dir)
|
|
print(f" {name} {info['fullver']} already on registry; skipping build")
|
|
return []
|
|
|
|
run(cfg.makepkg_cmd, cwd=build_dir, dry_run=cfg.dry_run)
|
|
if cfg.dry_run:
|
|
return []
|
|
|
|
artifacts = find_artifacts(build_dir, want_sig=cfg.sign)
|
|
pkgs = [a for a in artifacts if not a.name.endswith(".sig")]
|
|
if not pkgs:
|
|
raise RuntimeError(f"{name}: no *.pkg.tar.* produced")
|
|
print(f" built: {', '.join(a.name for a in pkgs)}")
|
|
return artifacts
|
|
|
|
|
|
def build_config(args: argparse.Namespace) -> Config:
|
|
makepkg_cmd = ["makepkg", "-s", "--noconfirm", "--needed"]
|
|
if args.cleanbuild:
|
|
makepkg_cmd.append("--cleanbuild")
|
|
if args.skip_pgp:
|
|
makepkg_cmd.append("--skippgpcheck")
|
|
if args.sign:
|
|
makepkg_cmd.append("--sign")
|
|
return Config(
|
|
repo_root=args.repo_root.resolve(),
|
|
workdir=args.workdir.resolve(),
|
|
makepkg_cmd=makepkg_cmd,
|
|
sign=args.sign,
|
|
keep_going=args.keep_going,
|
|
dry_run=args.dry_run,
|
|
publish=not args.no_publish,
|
|
replace=args.replace,
|
|
force=args.force,
|
|
registry_url=args.registry_url,
|
|
owner=args.owner,
|
|
arch_repo=args.arch_repo,
|
|
auth_user=args.auth_user or args.owner,
|
|
token=args.token,
|
|
ignore=load_ignore(args.repo_root.resolve()),
|
|
)
|
|
|
|
|
|
def main(argv: List[str]) -> int:
|
|
ap = argparse.ArgumentParser(description="Build Arch packages and publish to Gitea")
|
|
ap.add_argument("--repo-root", type=Path, default=Path.cwd())
|
|
ap.add_argument("--workdir", type=Path, default=Path.cwd() / ".build")
|
|
ap.add_argument("--only", nargs="*", default=[])
|
|
ap.add_argument("--list-json", action="store_true")
|
|
ap.add_argument("--cleanbuild", action="store_true")
|
|
ap.add_argument("--skip-pgp", action="store_true",
|
|
default=env("AUTOPKG_SKIP_PGP", "") not in ("", "0", "false"))
|
|
ap.add_argument("--sign", action="store_true")
|
|
ap.add_argument("--keep-going", action="store_true")
|
|
ap.add_argument("--dry-run", action="store_true")
|
|
ap.add_argument("--no-publish", action="store_true")
|
|
ap.add_argument("--replace", action="store_true",
|
|
default=env("AUTOPKG_REPLACE", "") not in ("", "0", "false"))
|
|
ap.add_argument("--force", action="store_true",
|
|
default=env("AUTOPKG_FORCE", "") not in ("", "0", "false"))
|
|
ap.add_argument("--registry-url", default=env("AUTOPKG_REGISTRY_URL", env("GITHUB_SERVER_URL")))
|
|
ap.add_argument("--owner", default=env("AUTOPKG_OWNER", env("GITHUB_REPOSITORY_OWNER")))
|
|
ap.add_argument("--arch-repo", default=env("AUTOPKG_REPO", "arch"))
|
|
ap.add_argument("--auth-user", default=env("AUTOPKG_USER", env("GITHUB_ACTOR")))
|
|
ap.add_argument("--token", default=env("AUTOPKG_TOKEN", env("PACKAGE_TOKEN", env("GITHUB_TOKEN"))))
|
|
args = ap.parse_args(argv)
|
|
cfg = build_config(args)
|
|
|
|
if args.list_json:
|
|
print(json.dumps([p.name for p in discover_packages(cfg, args.only)]))
|
|
return 0
|
|
|
|
pkgs = discover_packages(cfg, args.only)
|
|
if not pkgs:
|
|
print("No package directories found.", file=sys.stderr)
|
|
return 2
|
|
print(f"Discovered {len(pkgs)} package(s): {', '.join(p.name for p in pkgs)}")
|
|
|
|
if cfg.publish and not cfg.dry_run:
|
|
missing = [n for n, v in (("registry-url", cfg.registry_url),
|
|
("owner", cfg.owner), ("token", cfg.token)) if not v]
|
|
if missing:
|
|
print(f"Cannot publish: missing {', '.join(missing)}", file=sys.stderr)
|
|
return 2
|
|
|
|
failures = []
|
|
for pkg_src in pkgs:
|
|
try:
|
|
artifacts = build_one(pkg_src, cfg)
|
|
if cfg.publish and artifacts and not cfg.dry_run:
|
|
for art in artifacts:
|
|
publish_file(art, cfg)
|
|
elif cfg.publish and cfg.dry_run:
|
|
print(f" (dry-run) would publish {pkg_src.name}")
|
|
except (subprocess.CalledProcessError, RuntimeError, OSError) as e:
|
|
print(f"✗ {pkg_src.name}: {e}", file=sys.stderr)
|
|
failures.append(pkg_src.name)
|
|
if not cfg.keep_going:
|
|
break
|
|
|
|
if failures:
|
|
print(f"\nFailed: {', '.join(failures)}", file=sys.stderr)
|
|
return 1
|
|
print("\nAll done.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main(sys.argv[1:]))
|