ports/Tools/scripts/update-plist.py

#!/usr/bin/env python

# update-plist.py - Adds and removes lines in pkg-plist(s) based on
# poudriere error logs.

# Inspired by: https://codeberg.org/tcberner/poudlist, by tcberner@,
# but this just requires base Python and doesn't use a config file.

# ----------------------------------------------------------------------------
# "THE BEER-WARE LICENSE" (Revision 42, (c) Poul-Henning Kamp):
# Jason E. Hale <jhale@FreeBSD.org> wrote this file. As long as you retain
# this notice you can do whatever you want with this stuff. If we meet some
# day, and you think this stuff is worth it, you can buy me a beer in return.
#
# Jason E. Hale
# ----------------------------------------------------------------------------

# MAINTAINER=    jhale@FreeBSD.org

# SYNOPSIS
#   update-plist.py [-h] [-E] [-p </path/to/portsdir>] </path/to/poudriere/error/log>

# DESCRIPTION
#   Adds and removes lines in pkg-plist(s) based on poudriere error logs.

#   It is recommend to run this tool after building a port with all options
#   enabled with either `poudriere bulk -t` or `poudriere testport` as it
#   will prune all stale plist entires, including those prefixed by '%%',
#   but it does recognize where it has already been run and won't try to
#   duplicate the previous effort.

#   Use ports-mgmt/hs-panopticum for a more in-depth and CPU-intensive
#   analysis than this little script could ever provide, but with some
#   human brains behind it, using this takes a fraction of the time for
#   most ports.

#   You will still need to manually verify that all of the PLIST_SUBs
#   are correct. This tool follows the poudriere logs to a tee, but
#   sometimes the PLIST_SUBs for a port are more aggressive than they
#   really should be as generated by 'make makeplist'.

# OPTIONS
#   -h, --help      print helps and then exits
#
#   -p, --portsdir  path to the ports tree to operate upon
#                   Overrides the PORTSDIR environment variable
#                   if defined, required otherwise.
#
#   -E, --extended  (experimental) try to work with ports with no pkg-plist
#                   or more complicated pkg-plist structures (PLIST+=foo).
#                   Writes pkg-plist.<pkgname>.add~ and pkg-plist.<pkgname>.rm~
#                   files to the port directory  manual intervention instead of
#                   skipping. The appended '~' is meant to prevent accidental
#                   commital of these temporary files.

# ENVIRONMENT
#   PORTSDIR        path to the ports tree to operate upon

# EXAMPLES
#   update-plist.py /usr/local/poudriere/data/logs/bulk/150amd64-area51/2026-03-19_22h36m33s/logs/errors
#   update-plist.py -p ~/src/area51/ports /usr/local/poudriere/data/logs/bulk/150amd64-area51/2026-03-19_22h36m33s/logs/errors/qt6-webengine-6.11.0.log

# BUGS
#   - Assumes that the existing pkg-plist is already somewhat properly sorted.

import sys
import re
import os
import argparse
import subprocess

def extract_plist_sub(log_content):
    sub_map = {}
    match = re.search(r"--PLIST_SUB--\n(.*?)\n--End PLIST_SUB--", log_content, re.DOTALL)
    if match:
        raw_subs = match.group(1)
        pairs = re.findall(r'(\w+)=("(?:[^"\\]|\\.)*"|[^\s]+)', raw_subs)
        for key, val in pairs:
            clean_val = val.strip('"')
            sub_map[f"%%{key}%%"] = "" if "@comment" in clean_val else clean_val
    return sub_map

def get_expanded_key(line, sub_map):
    expanded = line.strip()
    for macro in sorted(sub_map.keys(), key=len, reverse=True):
        if macro in expanded:
            expanded = expanded.replace(macro, sub_map[macro]).replace("//", "/")
    clean = re.sub(r"%%.*?%%", "", expanded)
    clean = re.sub(r"@\w+\s+", "", clean)
    return clean.strip().lstrip('/')

def get_plist_paths(ports_tree_root, origin):
    """Uses make to resolve all PLIST paths, handling multiple space-separated plists."""
    port_dir = os.path.join(ports_tree_root, origin)
    try:
        result = subprocess.check_output(
            ['make', '-C', port_dir, '-V', 'PLIST'],
            text=True,
            stderr=subprocess.DEVNULL
        ).strip()

        if result:
            paths = result.split()
            return [p if os.path.isabs(p) else os.path.join(port_dir, p) for p in paths]
    except Exception:
        pass

    return [os.path.join(port_dir, "pkg-plist")]

def process_log(log_path, ports_tree_root, extended=False):
    try:
        with open(log_path, 'r', encoding='utf-8', errors='ignore') as f:
            first_lines = []
            for _ in range(15):
                line = f.readline()
                if not line: break
                first_lines.append(line)
            f.seek(0)
            log_content = f.read()
    except Exception as e:
        sys.stderr.write(f"![Error] Could not read {log_path}: {e}\n")
        return

    if "Error: Plist issues found" not in log_content:
        return

    origin = None
    for line in first_lines:
        if "port directory:" in line:
            parts = line.split(":", 1)[1].strip().rstrip('/').split('/')
            if len(parts) >= 2:
                origin = os.path.join(parts[-2], parts[-1])
            break

    if not origin: return

    plist_paths = get_plist_paths(ports_tree_root, origin)
    existing_plists = [p for p in plist_paths if os.path.exists(p)]

    is_fallback_mode = False
    fallback_rm_path = None

    # Handle the case where no valid plists exist on disk
    if not existing_plists:
        if not extended:
            sys.stderr.write(f"![Skip] {origin}: No valid plist files found (port likely relies on PLIST_FILES). Try using with the '-E' flag.\n")
            return
        else:
            is_fallback_mode = True
            log_basename = os.path.basename(log_path)
            if log_basename.endswith(".log"):
                log_basename = log_basename[:-4]

            fallback_add_name = f"pkg-plist.{log_basename}.add~"
            fallback_rm_name = f"pkg-plist.{log_basename}.rm~"
            fallback_add_path = os.path.join(ports_tree_root, origin, fallback_add_name)
            fallback_rm_path = os.path.join(ports_tree_root, origin, fallback_rm_name)

            sys.stderr.write(f"-> Warning: {origin} has no existing plist. Manual intervention required.\n")
            existing_plists = [fallback_add_path]

    plist_data = {}
    all_existing_lines = set()

    for path in existing_plists:
        if os.path.exists(path):
            with open(path, 'r', encoding='utf-8') as f:
                lines = [l.strip() for l in f.readlines() if l.strip()]
                plist_data[path] = lines
                all_existing_lines.update(lines)
        else:
            # For the newly created pkg-plist.*.add~
            plist_data[path] = []

    sub_map = extract_plist_sub(log_content)
    orphans = re.findall(r"Error: Orphaned: (.*)", log_content)
    missing_from_log = set(re.findall(r"Error: Missing: (.*)", log_content))

    removed_count = 0
    added_count = 0

    if is_fallback_mode:
        # We don't have a source plist to remove from, so dump the missing lines to the rm file
        if missing_from_log:
            plist_data[fallback_rm_path] = sorted(list(missing_from_log))
            removed_count = len(missing_from_log)
    else:
        for path, lines in plist_data.items():
            new_lines = []
            for line in lines:
                clean_line = re.sub(r"%%.*?%%", "", line).strip().lstrip('/')
                if line in missing_from_log or clean_line in missing_from_log:
                    removed_count += 1
                    continue
                new_lines.append(line)
            plist_data[path] = new_lines

    # Target the FIRST file in our list for orphans (either the base plist or the unique .add~ file)
    base_plist = existing_plists[0]

    for orphan in sorted(orphans, key=lambda x: get_expanded_key(x, sub_map)):
        if orphan in all_existing_lines:
            continue

        orphan_key = get_expanded_key(orphan, sub_map)
        inserted = False

        for i, line in enumerate(plist_data[base_plist]):
            if get_expanded_key(line, sub_map) > orphan_key:
                plist_data[base_plist].insert(i, orphan)
                inserted = True
                added_count += 1
                break

        if not inserted:
            plist_data[base_plist].append(orphan)
            added_count += 1

        all_existing_lines.add(orphan)

    if added_count == 0 and removed_count == 0:
        sys.stderr.write(f"-> {origin}: Already up to date.\n")
        return

    # Suppress the multi-plist warning if we are writing to a fallback intervention file
    if len(existing_plists) > 1 and added_count > 0 and "pkg-plist.add" not in base_plist:
        base_name = os.path.basename(base_plist)
        sys.stderr.write(f"-> Warning: {origin} uses multiple plists. Orphans appended to {base_name}. Please verify manually.\n")

    sys.stderr.write(f"-> Updating {origin}... Done (+{added_count}/-{removed_count})\n")

    for path, lines in plist_data.items():
        with open(path, 'w', encoding='utf-8') as f:
            f.write("\n".join(lines) + "\n")

if __name__ == "__main__":
    env_portsdir = os.environ.get("PORTSDIR")
    is_required = not bool(env_portsdir)

    parser = argparse.ArgumentParser(description="Updates pkg-plist based on poudriere error logs.")
    parser.add_argument(
        "logpath",
        help="Poudriere log file or directory of logs"
    )
    parser.add_argument(
        "-p", "--portsdir",
        required=is_required,
        default=env_portsdir,
        help="Path to ports tree to operate upon (Overrides PORTSDIR env var if set)"
    )
    parser.add_argument(
        "-E", "--extended",
        action="store_true",
        help="(experimental) If a port has no pkg-plist or a more complicated pkg-plist structure, write orphaned/missing files to pkg-plist.<pkgname>.add~ and pkg-plist.<pkgname>.rm~ for manual review instead of skipping"
    )
    args = parser.parse_args()

    if os.path.isdir(args.logpath):
        for filename in sorted(os.listdir(args.logpath)):
            if filename.endswith(".log"):
                process_log(os.path.join(args.logpath, filename), args.portsdir, args.extended)
    else:
        process_log(args.logpath, args.portsdir, args.extended)