diff --git a/bconds.py b/bconds.py index 586904d..58fca91 100644 --- a/bconds.py +++ b/bconds.py @@ -1,10 +1,12 @@ +import argparse +import datetime import functools +import os import pathlib -import re -import subprocess import sys -from utils import CONFIG, log +from gitrepo import clone_into, refresh_gitrepo, patch_spec, refresh_or_clone +from utils import CONFIG, log, run KOJI_ID_FILENAME = 'koji.id' @@ -35,42 +37,7 @@ def bcond_cache_identifier(component_name, bcond_config, *, branch='', target='' branch = '' identifier = f'{component_name}:{withouts_id}:{withs_id}:{replacements_id}:{branch}:{target}' reverse_id_lookup[identifier] = bcond_config - return identifier - - -def run(*cmd, **kwargs): - kwargs.setdefault('check', True) - kwargs.setdefault('capture_output', True) - kwargs.setdefault('text', True) - return subprocess.run(cmd, **kwargs) - - -def clone_into(component_name, target, branch=''): - branch = branch or CONFIG['distgit']['branch'] - log(f' • Cloning {component_name} into "{target}"...', end=' ') - # I would like to use --depth=1 but that breaks rpmautospec - # https://pagure.io/fedora-infra/rpmautospec/issue/227 - run('fedpkg', 'clone', component_name, target, f'--branch={branch}') - log('done.') - - -def refresh_gitrepo(repopath, prune_exisitng=False): - log(f' • Refreshing "{repopath}" git repo...', end=' ') - git = 'git', '-C', repopath - head_before = run(*git, 'rev-parse', 'HEAD').stdout.rstrip() - run(*git, 'stash') - run(*git, 'reset', '--hard') - run(*git, 'pull') - head_after = run(*git, 'rev-parse', 'HEAD').stdout.rstrip() - if head_before == head_after: - if not prune_exisitng: - # we try to preserve the changes for local inspection, but if it fails, meh - run(*git, 'stash', 'pop', check=False) - log('already up to date.') - return False - else: - log(f'updated {head_before[:10]}..{head_after[:10]}.') - return True + return identifier def srpm_path(directory): @@ -87,28 +54,6 @@ def srpm_path(directory): return candidates[0] -def patch_spec(specpath, bcond_config): - log(f' • Patching {specpath.name}') - - run('git', '-C', specpath.parent, 'reset', '--hard') - - spec_text = specpath.read_text() - - lines = [] - for without in sorted(bcond_config.get('withouts', ())): - if without in bcond_config.get('withs', ()): - raise ValueError(f'Cannot have the same with and without: {without}') - lines.append(f'%global _without_{without} 1') - for with_ in sorted(bcond_config.get('withs', ())): - lines.append(f'%global _with_{with_} 1') - for macro, value in bcond_config.get('replacements', {}).items(): - spec_text = re.sub(fr'^(\s*)%(define|global)(\s+){macro}(\s+)\S.*$', - fr'\1%\2\g<3>{macro}\g<4>{value}', - spec_text, flags=re.MULTILINE) - lines.append(spec_text) - specpath.write_text('\n'.join(lines)) - - def submit_scratchbuild(repopath, target=''): command = ('fedpkg', 'build', '--scratch', '--srpm', f'--arches={CONFIG["architectures"]["koji"]}', '--nowait', '--background') @@ -139,10 +84,24 @@ def koji_status(koji_id): for line in output: if line.startswith('State: '): return line.split(' ')[-1] - raise RuntimeError('Carnot parse koji taskinfo output') + raise RuntimeError('Cannot parse koji taskinfo output') -def handle_exisitng_srpm(repopath, *, was_updated): +def koji_id_is_older_than_week(koji_id_path): + """ + Returns True if koji_id file was created earlier than a week ago. + We assume this is a treshold time after which Koji artifacts are deleted, + and we need to remove koji_id to retrigger the scratchbuild. + If koji_id was created within the last week, return False. + """ + koji_id_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(koji_id_path)) + one_week_ago = datetime.datetime.now() - datetime.timedelta(weeks=1) + if koji_id_mtime < one_week_ago: + return True + return False + + +def handle_existing_srpm(repopath, *, was_updated): srpm = srpm_path(repopath) if srpm and not was_updated: log(f' • Found {srpm.name}, will not rebuild; remove it to force me.') @@ -152,7 +111,7 @@ def handle_exisitng_srpm(repopath, *, was_updated): return None -def handle_exisitng_koji_id(repopath, *, was_updated): +def handle_existing_koji_id(repopath, *, was_updated): koji_id_path = repopath / KOJI_ID_FILENAME if koji_id_path.exists(): if was_updated: @@ -166,13 +125,17 @@ def handle_exisitng_koji_id(repopath, *, was_updated): f'removing {KOJI_ID_FILENAME}.') koji_id_path.unlink() return None + elif status == 'closed' and koji_id_is_older_than_week(koji_id_path): + log(f' • Koji task {koji_task_id} is older than one week, ' + f'there may be nothing to download; removing {KOJI_ID_FILENAME}.') + koji_id_path.unlink() else: log(f' • Koji task {koji_task_id} is {status}; ' - f'not rebulding (rm {KOJI_ID_FILENAME} to force).') + f'not rebuilding (rm {KOJI_ID_FILENAME} to force).') return koji_task_id -def scratchbuild_patched_if_needed(component_name, bcond_config, *, branch='', target=''): +def scratchbuild_patched_if_needed(component_name, bcond_config, *, branch='', target='', no_git_refresh=False): """ This will: 1. clone/fetch the given component_name package from Fedora to fedpkg_cache_dir @@ -187,18 +150,14 @@ def scratchbuild_patched_if_needed(component_name, bcond_config, *, branch='', t 6. return True if something was submitted to Koji """ repopath = pathlib.Path(CONFIG['cache_dir']['fedpkg']) / bcond_config['id'] - if repopath.exists(): - news = refresh_gitrepo(repopath) - else: - pathlib.Path(CONFIG['cache_dir']['fedpkg']).mkdir(exist_ok=True) - clone_into(component_name, repopath, branch=branch) - news = True - if srpm := handle_exisitng_srpm(repopath, was_updated=news): + news = refresh_or_clone(repopath, component_name, no_git_refresh=no_git_refresh, branch=branch) + + if srpm := handle_existing_srpm(repopath, was_updated=news): bcond_config['srpm'] = srpm return False - if koji_id := handle_exisitng_koji_id(repopath, was_updated=news): + if koji_id := handle_existing_koji_id(repopath, was_updated=news): bcond_config['koji_task_id'] = koji_id return False @@ -213,7 +172,7 @@ def scratchbuild_patched_if_needed(component_name, bcond_config, *, branch='', t return True -def download_srpm_if_possible(component_name, bcond_config): +def download_srpm_if_possible(bcond_config): """ This will: 1. inspect the bcond_config for srpm path and a koji build id @@ -230,13 +189,13 @@ def download_srpm_if_possible(component_name, bcond_config): command = ('koji', 'download-task', bcond_config['koji_task_id'], '--arch=src', '--noprogress') koji_output = run(*command, cwd=repopath).stdout.splitlines() if (l := len(koji_output)) != 1: - raise RuntimeError(f'Cannot parse koji download-task ouptut, expected 1 line, got {l}') + raise RuntimeError(f'Cannot parse koji download-task output, expected 1 line, got: {l}') srpm_filename = koji_output[0].split(' ')[-1] if not srpm_filename.endswith('.src.rpm'): - raise RuntimeError('Cannot parse koji download-task ouptut, expected a *.src.rpm filename, got {srpm_filename}') + raise RuntimeError(f'Cannot parse koji download-task output, expected a *.src.rpm filename, got: {srpm_filename}') srpm = repopath / srpm_filename if not srpm.exists(): - raise RuntimeError('Downloaded SRPM does not exist: {srpm}') + raise RuntimeError(f'Downloaded SRPM does not exist: {srpm}') bcond_config['srpm'] = srpm log(srpm_filename) return True @@ -255,7 +214,7 @@ def rpm_requires(rpm): return tuple(sorted({r for r in raw_requires if not r.startswith('rpmlib(')})) -def extract_buildrequires_if_possible(component_name, bcond_config): +def extract_buildrequires_if_possible(bcond_config): """ This will: 1. inspect the bcond_config for srpm path @@ -285,11 +244,31 @@ def build_reverse_id_lookup(): pass +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + '-R', '--no-git-refresh', + action='store_true', + default=False, + help="Don't refresh the gitrepo of each existing component, just send new components scratchbuilds and downloads srpms." + ) + parser.add_argument( + 'packages', + nargs='*', + help='Only fetch bconds for given package name(s).' + ) + return parser.parse_args() + + if __name__ == '__main__': + args = parse_args() + # build everything something_was_submitted = False for component_name, bcond_config in each_bcond_name_config(): - something_was_submitted |= scratchbuild_patched_if_needed(component_name, bcond_config) + if args.packages and component_name not in args.packages: + continue + something_was_submitted |= scratchbuild_patched_if_needed(component_name, bcond_config, no_git_refresh=args.no_git_refresh) # download everything until there's nothing downloaded # the idea is that while downloading, other tasks could finish @@ -299,10 +278,12 @@ def build_reverse_id_lookup(): something_was_downloaded = False # while we were downloading, we could have finished Koji builds for pkg, bcond_configs in CONFIG['bconds'].items(): + if args.packages and pkg not in args.packages: + continue for bcond_config in bcond_configs: if 'buildrequires' not in bcond_config: - something_was_downloaded |= download_srpm_if_possible(component_name, bcond_config) - if extract_buildrequires_if_possible(component_name, bcond_config): + something_was_downloaded |= download_srpm_if_possible(bcond_config) + if extract_buildrequires_if_possible(bcond_config): extracted_count += 1 koji_status.cache_clear() diff --git a/build.py b/build.py index 9d5383c..0425b10 100644 --- a/build.py +++ b/build.py @@ -1,14 +1,9 @@ import pathlib import sys -# this module reuses bconds functions heavily -# XXX move to a common module? -from bconds import clone_into, refresh_gitrepo, patch_spec, run - -# the following bcond things actually do stay there from bconds import reverse_id_lookup, build_reverse_id_lookup - -from utils import CONFIG +from gitrepo import clone_into, refresh_gitrepo, patch_spec, refresh_or_clone +from utils import CONFIG, run PATCHDIR = pathlib.Path('patches_dir') @@ -26,13 +21,8 @@ bootstrap = reverse_id_lookup[component_name] component_name, *_ = component_name.partition(':') - # XXX make a reusable function with just refresh_gitrepo/clone_into repopath = FEDPKG_CACHEDIR / component_name - if repopath.exists(): - refresh_gitrepo(repopath, prune_exisitng=True) - else: - FEDPKG_CACHEDIR.mkdir(exist_ok=True) - clone_into(component_name, repopath) + refresh_or_clone(repopath, component_name, prune_existing=True) specpath = repopath / f'{component_name}.spec' @@ -55,7 +45,7 @@ # Bump and commit only if we haven't already, XXX ability to force this head_commit_msg = run('git', '-C', repopath, 'log', '--format=%B', '-n1', 'HEAD').stdout.rstrip() - if False: # and bootstrap or head_commit_msg != message: + if bootstrap: # or head_commit_msg != message: run('rpmdev-bumpspec', '-c', message, '--userstring', CONFIG['distgit']['author'], specpath) run('git', '-C', repopath, 'commit', '--allow-empty', f'{component_name}.spec', '-m', message, '--author', CONFIG['distgit']['author']) @@ -63,11 +53,11 @@ run('git', '-C', repopath, 'push') run('fedpkg', 'build', '--fail-fast', '--nowait', '--target', CONFIG['koji']['target'], cwd=repopath) # '--background' - # XXX prune this directory becasue we don't want no thousands clones? + # XXX prune this directory because we don't want no thousands clones? # maybe we are not gonna need this? except Exception: print(sys.argv[1]) raise - # XXX prune this directory becasue we don't want no thousands clones? + # XXX prune this directory because we don't want no thousands clones? # maybe we are not gonna need this? diff --git a/config.toml b/config.toml index dc3cc48..0ba1950 100644 --- a/config.toml +++ b/config.toml @@ -4,36 +4,31 @@ ## build python3.N-1 together with python3.N without any bcond, verify python3.N-1 is nonmain python #- python3.N: # macros: -# _with_bootstrap: 1 # bump the release so it's greater even with ~bootstrap -# _without_rpmwheels: 1 -# _without_tests: 1 -# _without_optimizations: 1 +# %global _with_bootstrap 1 # bump the release so it's greater even with ~bootstrap +# %global _without_rpmwheels 1 +# %global _without_tests 1 +# %global _without_optimizations 1 #- gdb: # not blocking (yet) # macros: -# _without_python: 1 +# %global _without_python 1 #- python-flit-core: # macros: -# _with_bootstrap: 1 # bump the release so it's greater even with ~bootstrap +# %global _with_bootstrap 1 # bump the release so it's greater even with ~bootstrap #- python-packaging: # macros: -# _with_bootstrap: 1 # bump the release so it's greater even with ~bootstrap +# %global _with_bootstrap 1 # bump the release so it's greater even with ~bootstrap #- python-setuptools: # macros: -# _with_bootstrap: 1 # bump the release so it's greater even with ~bootstrap -#- python-wheel: -# macros: -# _with_bootstrap: 1 # bump the release so it's greater even with ~bootstrap +# %global _with_bootstrap 1 # bump the release so it's greater even with ~bootstrap #- python-pip: # macros: -# _without_tests: 1 -# _without_doc: 1 -#- python-setuptools: -# macros: -# _without_tests: 1 +# %global _without_tests 1 +# %global _without_man 1 #- pyparsing: # python BuildRequires systemtap-sdt-devel Requires pyparsing # macros: -# _without_tests: 1 -# _without_doc: 1 +# %global _without_tests 1 +# %global _without_doc 1 +# %global _without_extras 1 ## wait for gdb #- python3.N # VERIFY all of the above have correct Provides/Requires @@ -41,11 +36,12 @@ [koji] target = 'rawhide' +# target = 'f43-python' [distgit] branch = "rawhide" -commit_message = "Rebuilt for Python 3.12" -bootstrap_commit_message = "Bootstrap for Python 3.12" +commit_message = "Rebuilt for Python 3.14" +bootstrap_commit_message = "Bootstrap for Python 3.14" author = "Python Maint " [architectures] @@ -53,12 +49,12 @@ repoquery = "x86_64" # used for repository querying koji = "x86_64" # used to scratch build packages with bconds [deps] -old = ["python(abi) = 3.11", "libpython3.11.so.1.0()(64bit)", "libpython3.11d.so.1.0()(64bit)"] -new = ["python(abi) = 3.12", "libpython3.12.so.1.0()(64bit)", "libpython3.12d.so.1.0()(64bit)"] +old = ["python(abi) = 3.13", "libpython3.13.so.1.0()(64bit)", "libpython3.13d.so.1.0()(64bit)"] +new = ["python(abi) = 3.14", "libpython3.14.so.1.0()(64bit)", "libpython3.14d.so.1.0()(64bit)"] [components] -excluded = ["python3.11", "python3.12"] -extra = ["python3-docs"] +excluded = ["python3.13", "python3.14", "avogadro2-libs", "kmymoney"] +extra = [] [cache_dir] dnf = "_dnf_cache_dir" @@ -68,25 +64,24 @@ fedpkg = "_fedpkg_cache_dir" [[repos.rawhide]] repoid = "rawhide" # metalink = "https://mirrors.fedoraproject.org/metalink?repo=rawhide&arch=$basearch" -baseurl = ["https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20230705.n.0/compose/Everything/$basearch/os/"] +baseurl = ["https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20250610.n.0/compose/Everything/$basearch/os/"] metadata_expire = 600000000 [[repos.rawhide]] repoid = "rawhide-source" # metalink = "https://mirrors.fedoraproject.org/metalink?repo=rawhide-source&arch=$basearch" -baseurl = ["https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20230705.n.0/compose/Everything/source/tree/"] +baseurl = ["https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20250610.n.0/compose/Everything/source/tree/"] metadata_expire = 600000000 [[repos.target]] -repoid = "python3.12" -#baseurl = ["https://copr-be.cloud.fedoraproject.org/results/@python/python3.12/fedora-rawhide-$basearch/"] -baseurl = ["http://kojipkgs.fedoraproject.org/repos/f39-build/latest/$basearch/"] +repoid = "python3.14" +# baseurl = ["https://copr-be.cloud.fedoraproject.org/results/@python/python3.14/fedora-rawhide-$basearch/"] +baseurl = ["http://kojipkgs.fedoraproject.org/repos/f44-build/latest/$basearch/"] metadata_expire = 60 [bconds] [[bconds.python-setuptools]] withs = ["bootstrap"] -withouts = ["tests"] [[bconds.python-packaging]] withs = ["bootstrap"] @@ -95,7 +90,7 @@ withs = ["bootstrap"] withs = ["bootstrap"] [[bconds.python-pip]] -withouts = ["tests", "doc"] +withouts = ["tests", "man"] [[bconds.python-six]] withouts = ["tests"] @@ -110,7 +105,7 @@ withouts = ["tests"] withouts = ["docs", "tests"] [[bconds.python-chardet]] -withouts = ["doc_pdf"] +withouts = ["doc"] [[bconds.python-pbr]] withs = ["bootstrap"] @@ -171,13 +166,13 @@ withouts = ["tests"] withouts = ["timeout", "tests", "docs"] [[bconds.python-virtualenv]] -withouts = ["tests"] +withs = ["bootstrap"] [[bconds.babel]] withs = ["bootstrap"] [[bconds.python-jinja2]] -withouts = ["docs"] +withouts = ["docs", "asyncio_tests"] [[bconds.python-sphinx_rtd_theme]] withs = ["bootstrap"] @@ -186,31 +181,13 @@ withs = ["bootstrap"] withs = ["bootstrap"] [[bconds.python-urllib3]] -withouts = ["tests"] +withouts = ["tests", "extradeps"] [[bconds.python-requests]] withouts = ["tests"] -[[bconds.python-sphinxcontrib-applehelp]] -withouts = ["check"] - -[[bconds.python-sphinxcontrib-devhelp]] -withouts = ["check"] - -[[bconds.python-sphinxcontrib-htmlhelp]] -withouts = ["check"] - -[[bconds.python-sphinxcontrib-jsmath]] -withouts = ["check"] - -[[bconds.python-sphinxcontrib-qthelp]] -withouts = ["check"] - -[[bconds.python-sphinxcontrib-serializinghtml]] -withouts = ["check"] - [[bconds.python-sphinx]] -withouts = ["tests", "websupport"] +withouts = ["tests", "sphinxcontrib"] [[bconds.python-jedi]] withouts = ["tests"] @@ -227,9 +204,6 @@ withouts = ["optional_tests"] [[bconds.python-soupsieve]] withouts = ["tests"] -[[bconds.python-towncrier]] -withouts = ["tests"] - [[bconds.python-pytest-asyncio]] withouts = ["tests"] @@ -242,17 +216,8 @@ withouts = ["tests"] [[bconds.python-async-timeout]] withouts = ["tests"] -[[bconds.python-trio]] -withouts = ["tests"] - -[[bconds.python-Automat]] -withouts = ["tests"] - -[[bconds.python-invoke]] -withouts = ["tests"] - [[bconds.python-jupyter-client]] -withouts = ["doc","tests"] +withouts = ["tests"] [[bconds.python-jupyter-server]] withouts = ["tests"] @@ -264,7 +229,7 @@ withouts = ["check"] withouts = ["check","doc"] [[bconds.python-ipykernel]] -withouts = ["tests","doc"] +withouts = ["tests"] [[bconds.pybind11]] withouts = ["tests"] @@ -284,36 +249,24 @@ withouts = ["tests"] [[bconds.freeipa-healthcheck]] withouts = ["tests"] -[[bconds.python-zbase32]] -withs = ["bootstrap"] - [[bconds.python-Traits]] withs = ["bootstrap"] -[[bconds.python-lit]] -withouts = ["check"] - [[bconds.python-pcodedmp]] withs = ["bootstrap"] [[bconds.python-libcst]] -withouts = ["tests", "docs"] +withs = ["bootstrap"] [[bconds.python-databases]] withs = ["bootstrap"] -[[bconds.python-molecule]] -withouts = ["doc"] - [[bconds.scipy]] withouts = ["pythran"] [[bconds.python-pandas]] withs = ["bootstrap"] -[[bconds.grpc]] -withs = ["bootstrap"] - [[bconds.python-fasteners]] withouts = ["tests"] @@ -322,7 +275,7 @@ withs = ["bootstrap"] withouts = ["docs"] [[bconds.python-zope-interface]] -withouts = ["tests", "docs"] +withouts = ["docs"] [[bconds.python-tqdm]] withouts = ["tests"] @@ -334,7 +287,7 @@ withouts = ["tests"] withouts = ["tests"] [[bconds.python-geopandas]] -withouts = ["tests"] +withs = ["bootstrap"] [[bconds.python-astropy]] withouts = ["check"] @@ -345,14 +298,13 @@ withouts = ["tests"] [[bconds.python-aiohttp]] withouts = ["tests"] -[[bconds.python-pep517]] -withouts = ["tests"] - [[bconds.python-azure-core]] withouts = ["tests"] -[[bconds.python-azure-common]] -withouts = ["tests"] +# this bcond is disabled in spec on purpose +# it might be enabled in the future, so leaving this here, commend out +#[[bconds.python-azure-common]] +#withouts = ["tests"] [[bconds.python-azure-mgmt-core]] withouts = ["tests"] @@ -361,10 +313,10 @@ withouts = ["tests"] withouts = ["tests"] [[bconds.python-typeguard]] -withouts = ["doc_pdf"] +withs = ["bootstrap"] [[bconds.python-tox]] -withouts = ["tests"] +withs = ["bootstrap"] [[bconds.python-pytest-rerunfailures]] withouts = ["tests"] @@ -381,9 +333,6 @@ withs = ["bootstrap"] [[bconds.python-msrest]] withouts = ["tests"] -[[bconds.numpy]] -withouts = ["tests"] - [[bconds.python-oletools]] withs = ["bootstrap"] @@ -391,8 +340,17 @@ withs = ["bootstrap"] withouts = ["tests"] [[bconds.fonttools]] +withouts = ["tests", "plot_extra", "symfont_extra", "ufo_extra", "woff_extra", "graphite_extra", "interpolatable_extra"] + +[[bconds.conda]] withouts = ["tests"] +[[bconds.python-conda-libmamba-solver]] +withouts = ["tests"] + +[[bconds.python-conda-index]] +withs = ["bootstrap"] + [[bconds.python-conda-package-streaming]] withs = ["bootstrap"] @@ -405,12 +363,6 @@ withouts = ["tests"] [[bconds.python-dns]] withouts = ["trio", "curio", "doh"] -[[bconds.python-poetry-plugin-export]] -withouts = ["bootstrap"] - -[[bconds.poetry]] -withs = ["bootstrap"] - [[bconds.python-networkx]] withs = ["bootstrap"] @@ -421,10 +373,13 @@ withouts = ["soupsieve", "tests"] withouts = ["extras"] [[bconds.python-constantly]] -withouts = ["tests"] +withs = ["bootstrap"] -[[bconds.gdb]] -replacements = {_without_python = "1"} +# this is intentionally commented out +# 1) is is part of the initial bootstrap +# 2) replacement does not work, the macro isn't there (needs to be added) +#[[bconds.gdb]] +#replacements = {_without_python = "1"} [[bconds.python-pysocks]] replacements = {with_python3_tests = "0"} @@ -448,10 +403,184 @@ withouts = ["plugins"] withouts = ["tests"] [[bconds.python-tox-current-env]] -withouts = ["tests"] +withs = ["bootstrap"] [[bconds.python-pikepdf]] -withs = ["docs", "tests"] +withouts = ["docs", "tests"] + +[[bconds.python-executing]] +withs = ["bootstrap"] + +[[bconds.python-editables]] +withouts = ["doc"] + +[[bconds.python-jsonschema-specifications]] +replacements = {with_doc = "0"} + +[[bconds.python-openstackclient]] +replacements = {with_doc = "0"} + +[[bconds.python-oslo-config]] +replacements = {repo_bootstrap = "1"} + +[[bconds.python-neutronclient]] +replacements = {with_doc = "0"} + +[[bconds.python-glanceclient]] +replacements = {with_doc = "0"} + +[[bconds.python-domdf-python-tools]] +withouts = ["tests"] + +[[bconds.python-tiny-proxy]] +withouts = ["tests"] + +[[bconds.python-werkzeug]] +withouts = ["tests"] + +[[bconds.python-netaddr]] +withouts = ["docs"] + +[[bconds.python-dirty-equals]] +withs = ["bootstrap"] + +[[bconds.python-snakemake-interface-storage-plugins]] +withs = ["bootstrap"] + +[[bconds.python-snakemake-interface-executor-plugins]] +withs = ["bootstrap"] + +[[bconds.python-snakemake-interface-report-plugins]] +withs = ["bootstrap"] + +[[bconds.snakemake]] +withs = ["bootstrap"] + +[[bconds.python-sphinx-theme-builder]] +withs = ["bootstrap"] + +[[bconds.python-httpx]] +withouts = ["tests"] + +[[bconds.python-oslo-i18n]] +replacements = {with_doc = "0"} + +[[bconds.python-pymssql]] +withouts = ["tests"] + +[[bconds.python-authlib]] +withouts = ["tests"] + +[[bconds.python-oslotest]] +withs = ["repo_bootstrap"] + +[[bconds.python-gunicorn]] +withouts = ["extras"] + +[[bconds.python-execnet]] +withouts = ["optional_test_deps"] + +[[bconds.python-threadpoolctl]] +withouts = ["check"] + +[[bconds.python-tabulate]] +withs = ["bootstrap"] + +[[bconds.python-repoze-sphinx-autointerface]] +withouts = ["tests"] + +[[bconds.python-zope-exceptions]] +withouts = ["tests"] + +[[bconds.python-deepdiff]] +withouts = ["tests"] + +[[bconds.python-sentry-sdk]] +withouts = ["tests"] + +[[bconds.python-dask]] +withs = ["bootstrap"] + +[[bconds.python-build]] +withouts = ["uv"] + +[[bconds.meson]] +withouts = ["check"] + +[[bconds.python-gmpy2]] +withouts = ["tests"] + +[[bconds.python-humanfriendly]] +withs = ["bootstrap"] + +[[bconds.python-cascadio]] +withs = ["bootstrap"] + +[[bconds.python-pint]] +withouts = ["xarray"] + +[[bconds.python-qcengine]] +withouts = ["tests"] + +[[bconds.python-astropy-iers-data]] +withouts = ["tests"] + +[[bconds.python-pymongo]] +withs = ["bootstrap"] + +[[bconds.python-pyogrio]] +withs = ["bootstrap"] + +[[bconds.python-fastapi]] +withs = ["bootstrap"] + +[[bconds.python-pydantic-core]] +withouts = ["inline_snapshot_tests"] + +[[bconds.python-optking]] +withouts = ["tests"] + +[[bconds.python-django5]] +withouts = ["tests"] + +[[bconds."python-django4.2"]] +withouts = ["tests"] + +[[bconds.python-pyproject-hooks]] +withouts = ["tests"] + +[[bconds.python-Automat]] +withouts = ["doc"] + +[[bconds.python-psutil]] +withouts = ["xdist"] + +[[bconds.python-apkinspector]] +withs = ["bootstrap"] + +[[bconds.python-androguard]] +withs = ["bootstrap"] + +[[bconds.python-service-identity]] +withouts = ["docs"] + +[[bconds.mkdocs-material]] +withs = ["bootstrap"] + +[[bconds.python-mkdocs-material-extensions]] +withouts = ["tests"] + +[[bconds.python-zope-testing]] +withouts = ["tests"] + +[[bconds.python-xnat]] +withouts = ["tests"] + +[[bconds.python-tablib]] +withs = ["bootstrap"] + +[[bconds.llvm]] +withouts = ["pgo"] # [[bconds.OpenColorIO]] # %global bootstrap 1 diff --git a/gitrepo.py b/gitrepo.py new file mode 100644 index 0000000..ec9580e --- /dev/null +++ b/gitrepo.py @@ -0,0 +1,76 @@ +""" +This module contains helper functions to manipulate with distgit repositories: +clone them, refresh their local copies and patch the specfiles. +""" + +import re + +from utils import CONFIG, log, run + + +def clone_into(component_name, target, branch=''): + branch = branch or CONFIG['distgit']['branch'] + log(f' • Cloning {component_name} into "{target}"...', end=' ') + # I would like to use --depth=1 but that breaks rpmautospec + # https://pagure.io/fedora-infra/rpmautospec/issue/227 + run('fedpkg', 'clone', component_name, target, f'--branch={branch}') + log('done.') + + +def refresh_gitrepo(repopath, prune_existing=False): + log(f' • Refreshing "{repopath}" git repo...', end=' ') + git = 'git', '-C', repopath + head_before = run(*git, 'rev-parse', 'HEAD').stdout.rstrip() + run(*git, 'stash') + run(*git, 'reset', '--hard') + run(*git, 'pull') + head_after = run(*git, 'rev-parse', 'HEAD').stdout.rstrip() + if head_before == head_after: + if not prune_existing: + # we try to preserve the changes for local inspection, but if it fails, meh + run(*git, 'stash', 'pop', check=False) + log('already up to date.') + return False + else: + log(f'updated {head_before[:10]}..{head_after[:10]}.') + return True + + +def patch_spec(specpath, bcond_config): + log(f' • Patching {specpath.name}') + + run('git', '-C', specpath.parent, 'reset', '--hard') + + spec_text = specpath.read_text() + + lines = [] + for without in sorted(bcond_config.get('withouts', ())): + if without in bcond_config.get('withs', ()): + raise ValueError(f'Cannot have the same with and without: {without}') + lines.append(f'%global _without_{without} 1') + for with_ in sorted(bcond_config.get('withs', ())): + lines.append(f'%global _with_{with_} 1') + for macro, value in bcond_config.get('replacements', {}).items(): + spec_text = re.sub(fr'^(\s*)%(define|global)(\s+){macro}(\s+)\S.*$', + fr'\1%\2\g<3>{macro}\g<4>{value}', + spec_text, flags=re.MULTILINE) + lines.append(spec_text) + specpath.write_text('\n'.join(lines)) + + +def refresh_or_clone(repopath, component_name, *, prune_existing=False, no_git_refresh=False, branch=''): + """ + Returns True if there's new contents of the repository. + Returns False if the content of the repository remains the same + or if no_git_refresh option is set to True + (we skip the repository update and assume nothing has changed). + """ + if repopath.exists(): + if no_git_refresh: + return False + else: + return refresh_gitrepo(repopath, prune_existing=prune_existing) + else: + repopath.parent.mkdir(exist_ok=True) + clone_into(component_name, repopath, branch=branch) + return True diff --git a/jobs.py b/jobs.py index 2782d5a..3eeff6f 100644 --- a/jobs.py +++ b/jobs.py @@ -1,5 +1,6 @@ import collections import functools +import os import sys from sacks import MULTILIB, rawhide_sack, target_sack @@ -100,10 +101,12 @@ def packages_built(new_deps, *, excluded_components=()): return components -def are_all_done(*, packages_to_check, all_components, components_done, blocker_counter, loop_detector): +def are_all_done(*, packages_to_check, all_components, components_done, blocker_counter, loop_detector, missing_packages): """ Given a collection of (binary) packages_to_check, and dicts of all_components and components_done, returns True if ALL packages_to_check are considered "done" (i.e. installable). + + missing_packages maps component names to sets of missing package names. """ relevant_components = ReverseLookupDict() for pkg in packages_to_check: @@ -120,30 +123,42 @@ def are_all_done(*, packages_to_check, all_components, components_done, blocker_ for required_package in required_packages: has_older = False for done_package in components_done.get(relevant_component, ()): + found = False # The done packages are from different repo and might have different EVR # Hence, we only compare the names # For Copr rebuilds, the Copr EVR must be >= Fedora EVR # For koji rebuilds, this will be always true anyway - # XXX cython was renamed after the rebuild - if done_package.name == required_package.name or required_package.name == 'python3-Cython': - #if not done_package.evr_lt(required_package): + if done_package.name == required_package.name: + # if not done_package.evr_lt(required_package): if True: log(f' ✔ {required_package.name}') break else: has_older = True + else: + # Check the virtual provides - maybe one of them matches what we look for + for provide in done_package.provides: + if provide.name == required_package.name: + log(f' ✔ {required_package.name}') + found = True + break + if found: + break else: if has_older: log(f' ✗ {required_package.name} (older EVR available)') else: log(f' ✗ {required_package.name}') + + missing_packages[component].add(required_package.name) + all_available = False count_component = True if count_component: blocker_counter['general'][relevant_component] += 1 blocking_components.add(relevant_component) if len(blocking_components) == 1: - blocker_counter['single'][blocking_components.pop()] += 1 + blocker_counter['single'][next(iter(blocking_components))] += 1 elif 1 < len(blocking_components) < 10: # this is an arbitrarily chosen number to avoid cruft blocker_counter['combinations'][tuple(sorted(blocking_components))] += 1 loop_detector[component] = sorted(blocking_components) @@ -157,6 +172,7 @@ def _sort_loop(loop): def _detect_loop(loop_detector, probed_component, depchain, loops, seen): for component in loop_detector[probed_component]: + recursedown = component not in seen seen.add(component) if component in CONFIG['bconds']: # we assume bconds are manually crafted not to have loops @@ -166,7 +182,8 @@ def _detect_loop(loop_detector, probed_component, depchain, loops, seen): if component in depchain: loops.add(_sort_loop(depchain[depchain.index(component):])) continue - _detect_loop(loop_detector, component, depchain + [component], loops, seen) + if recursedown: + _detect_loop(loop_detector, component, depchain + [component], loops, seen) def report_blocking_components(loop_detector): loops = set() @@ -174,10 +191,30 @@ def report_blocking_components(loop_detector): for component in loop_detector: if component not in seen: _detect_loop(loop_detector, component, [component], loops, seen) + seen.add(component) log('\nDetected dependency loops:') for loop in sorted(loops, key=lambda t: -len(t)): log(' • ' + ' → '.join(loop)) + +def get_component_status_info(component, missing_packages, components): + """Generate status information for a component explaining why it's blocked.""" + if component in components: + if component in missing_packages: + missing_deps = missing_packages[component] + if missing_deps: + missing_deps_str = ", ".join(sorted(missing_deps)[:3]) + if len(missing_deps) > 3: + missing_deps_str += "..." + return f" (blocked by: {missing_deps_str})" + else: + return " (blocked for unknown reason)" # This should never happen, but keep it for debugging + else: + return " (ready)" # This should never happen, but keep it for debugging + else: + return f" (build failed)" + + if __name__ == '__main__': # this is spaghetti code that will be split into functions later: from resolve_buildroot import resolve_buildrequires_of, resolve_requires @@ -196,6 +233,8 @@ def report_blocking_components(loop_detector): } loop_detector = {} + missing_packages = collections.defaultdict(set) # requiring_component -> missing packages + for component in components: if len(sys.argv) > 1 and component not in sys.argv[1:]: continue @@ -204,52 +243,62 @@ def report_blocking_components(loop_detector): component_buildroot = resolve_buildrequires_of(component) except ValueError as e: log(f'\n ✗ {e}') - continue + number_of_resolved = None + ready_to_rebuild = False + else: + number_of_resolved = len(component_buildroot) - ready_to_rebuild = are_all_done( - packages_to_check=set(component_buildroot) & binary_rpms, - all_components=components, - components_done=components_done, - blocker_counter=blocker_counter, - loop_detector=loop_detector, - ) + ready_to_rebuild = are_all_done( + packages_to_check=set(component_buildroot) & binary_rpms, + all_components=components, + components_done=components_done, + blocker_counter=blocker_counter, + loop_detector=loop_detector, + missing_packages=missing_packages, + ) if ready_to_rebuild: - # XXX make this configurable - if component not in components_done: + if os.environ.get('PRINT_ALL') or component not in components_done: print(component) elif component in CONFIG['bconds']: for bcond_config in CONFIG['bconds'][component]: bcond_config['id'] = bcond_cache_identifier(component, bcond_config) log(f'• {component} not ready and {bcond_config["id"]} bcond found, will check that one') if 'buildrequires' not in bcond_config: - extract_buildrequires_if_possible(component, bcond_config) + extract_buildrequires_if_possible(bcond_config) if 'buildrequires' in bcond_config: try: component_buildroot = resolve_requires(tuple(sorted(bcond_config['buildrequires']))) except ValueError as e: log(f'\n ✗ {e}') continue + if number_of_resolved == len(component_buildroot): + # XXX when this happens, the bcond might be bogus + # figure out a way to present that information + pass ready_to_rebuild = are_all_done( packages_to_check=set(component_buildroot) & binary_rpms, all_components=components, components_done=components_done, blocker_counter=blocker_counter, loop_detector=loop_detector, + missing_packages=missing_packages, ) if ready_to_rebuild: - if component not in components_done: + if os.environ.get('PRINT_ALL') or component not in components_done: print(bcond_config['id']) else: log(f' • {bcond_config["id"]} bcond SRPM not present yet, skipping') log('\nThe 50 most commonly needed components are:') for component, count in blocker_counter['general'].most_common(50): - log(f'{count:>5} {component}') + status_info = get_component_status_info(component, missing_packages, components) + log(f'{count:>5} {component:<35} {status_info}') log('\nThe 20 most commonly last-blocking components are:') for component, count in blocker_counter['single'].most_common(20): - log(f'{count:>5} {component}') + status_info = get_component_status_info(component, missing_packages, components) + log(f'{count:>5} {component:<35} {status_info}') log('\nThe 20 most commonly last-blocking small combinations of components are:') for components, count in blocker_counter['combinations'].most_common(20): diff --git a/progress.pkgs b/progress.pkgs index a435983..760ff2d 100644 --- a/progress.pkgs +++ b/progress.pkgs @@ -1,107 +1,392 @@ -andriller -androguard -androwarn -awscli -brd -btest -condor -cvc4 -swid-tools -eric -fail2ban -fedora-gather-easyfix -imgbased -python-mathics3 -pico-wizard -pyflowtools -python-acora -python-adext -python-aioambient -python-aioguardian -python-aiozeroconf -python-airthings -python-alarmdecoder -python-ansi -python-ansible-pygments -python-async-generator -awake -python-blockdiag -python-bluepy -python-box -python-certbot-dns-cloudxns -python-click-spinner -python-cypari2 -python-cypy -dionaea -python-django-contact-form -python-django-robots -python-django3 -python-dominate -python-editdistance-s -python-flask-bootstrap -python-fpylll -python-gccinvocation -python-geomet -python-grako -python-graphitesend -python-igor -python-ipgetter -python-jep -python-jsonrpc-server -python-lacrosse -python-lark-parser -python-lasagne -python-lazr-smtptest -lensfun -python-leveldb -python-liblarch -libssh2-python -python-logging-tree -python-logutils -python-mpd2 -mraa -python-networkmanager -python-nipy -python-nose_fixes -python-notario -openshadinglanguage -python-optcomplete -python-podman-api -python-pvc -python-py9p -python-pyaes -python-pybv -python-pydiffx -pyftpdlib -libgpuarray -python-pymc3 -python-pyngus -python-pyoptical -python-pyside2 -python-pytelegrambotapi -python-pytest-flake8 -python-pytest-metadata -python-pytest-virtualenv -python3-script -python-simpleparse -python-simplewrap +artifacts +asciinema +asv +cairo-dock-plug-ins +cantera +cantoolz +cranc +python-dcrpm +fros +hashid +hypershell +ipa-hcc +linode-cli +mom +mygnuhealth +netstat-monitor +neuron +odcs +pagure +parsero +past-time +pdfposter +poezio +pypykatz +pysubnettree +python-play-scraper +python-ufl +ATpy +python-AWSIoTPythonSDK +GeographicLib +python-Naked +python-PyLEMS +python-XStatic +python-XStatic-Angular +python-XStatic-Angular-Bootstrap +python-XStatic-Angular-Gettext +python-XStatic-Angular-lrdragndrop +python-XStatic-Hogan +python-XStatic-JQuery-Migrate +python-XStatic-JQuery-TableSorter +python-XStatic-JQuery-quicksearch +python-XStatic-JSEncrypt +python-XStatic-Jasmine +python-XStatic-Magic-Search +python-XStatic-QUnit +python-XStatic-Rickshaw +python-XStatic-Spin +python-XStatic-smart-table +python-XStatic-termjs +python-accuweather +python-adafruit-pureio +python-adjustText +python-afsapi +python-aioasuswrt +python-aiocmd +python-aioeafm +python-aioesphomeapi +python-aiogqlc +python-aiohomekit +python-aiohttp-sse-client +python-aiohue +python-aioiotprov +python-aiokafka +python-aiolifx +python-aiomultiprocess +python-aionotion +python-aiopg +python-aiorestapi +python-aiosecretsdump +python-aiosmb +python-aiosnmp +python-aiowinreg +python-airspeed +python-ajpy +python-amply +python-ana +python-ansicolors +python-aresponses +python-astral +python-atomicwrites +python-autograd +python-avocado +python-aws-sam-translator +python-awsiotsdk +python-baluhn +python-betamax-matchers +python-betamax-serializers +python-bioframe +python-boututils +python-cachy +python-chalice +python-check-manifest +python-chirpstack-api +python-ci-info +cjdns +python-click-help-colors +python-cliff-tablib +python-cloudant +python-coapthon3 +python-commandparse +python-compal +conda-build +python-connect-box +python-coronavirus +python-crcelk +credslayer +python-cx-oracle +python-cxxfilt +python-cyipopt +python-daikin +python-danfossair +python-dataclassy +python-dateutils +python-datrie +python-deconz +python-devolo-home-control-api +python-dfdatetime +python-dictdumper +python-didl-lite +python-dingz +distro-info +python-django-contrib-comments +python-django-threadedcomments +python-django4.2:tests:::: +dlib +dnsgen +python-dnslib +python-docx +dput-ng +python-dtfabric +python-earthpy +python-easyco +python-edimax +python-editdistance +python-elephant +python-enrich +python-enturclient +python-epi +python-epson-projector +python3-exiv2 +python-fiat +python-firkin +flann +python-formulaic +python-friendlyloris +python-fuzzywuzzy +python-gammu +python-gekitchen +python-gios +python-git-url-parse +python-glances-api +python-gphoto2 +python-graphql-relay +python-grip +python-hass-data-detective +python-hatasmota +python-hdate +python-hdf5storage +python-hgdistver +python-hikvision +hokuyoaist +python-hole +python-homeconnect +python-homeworks +python-hstspreload +python-hudman +python-imagehash +python-instant +python-intern +python-ipgetter2 +python-iso-639 +python-itanium_demangler +python-janus +python-javalang +python-javaobj +python-jschema-to-python +python-json2table +python-jsonpath-rw-ext +python-junit_xml +python-keepassxc-browser +kerberoast +python-kismet-rest +python-korean-lunar-calendar +python-lazy-ops +ldapdomaindump +python-ldappool +ldeep +libfreenect +libnl3 +python-linkheader +python-livereload +python-lqrt +python-makeelf +python-markups +python-marshmallow-enum +python-masscan +python-metno +metrics2mqtt +python-microfs +python-mido +python-minidump +python-mizani +python-mongoquery +python-mpd +python-mrcrowbar +python-msldap +python-mystrom +python-nanoid +python-natlas-libnmap +python-ndeflib +python-nessus-file-reader +python-netdata +python-network-runner +python-neurodsp +python-nrf24 +python-nudatus +python-nuheat +python-octave-kernel +python-odml +python-onigurumacffi +python-openctm +python-opendata-transport +python-opensensemap-api +python-os-testr +python-outdated +python-pamela +python-pastel +python-pifpaf +python-pingouin +python-pkginfo2 +python-plac +python-plaintable +python-plotly +python-plugnplay +python-plumbum +python-policyuniverse +python3-postgresql +python-poyo +python-probeinterface +python-promise +pwncat +python-py-algorand-sdk +python-py27hash +python-py2pack +python-pyABF +python-pydes +python-pyactivetwo +python-pyairnow +python-pybalboa +python-pycatch22 +python-pycoingecko +python-pycomm3 +python-pydotplus +python-pyeclib +python-pyemby +python-pyemd +python-pyfim +python-pygatt +python-pygrocy +python-pyhomematic +python-pyi2cflash +python-pyinels +python-pyiqvia +python-pylatex +python-PyLink +python-pylotoncycle +python-pymata-express +python-pymochad +python-pymod2pkg +python-pynamodb +python-pynuvo +python-pyopenuv +python-pyotgw +python-pypcapkit +python-pypck +python-pyprocdev +python-pyqtchart +python-pyriemann +python-pysaml2 +pyserial-asyncio +python-pysmb +python-pysmt +python-pysqueezebox +python-pytapo +python-pytest-error-for-skips +python-pytest-ordering +python-pytest-twisted +python-pytimeparse +python-pyunicorn +python-pyupgrade +python-pyvit +python-pyxdf +python-pyxid +python-q +python-qcengine:tests:::: +rabbitvcs +python-rak811 +python-ramalama +python-random2 +python-rangeparser +python-ratinabox +python-rawkit +python-read-roi +python-readlike +python-recordclass +python-regenmaschine +python-registry +python-remoto +python-reparser +python-requests-credssp +restview +python-retrying +python-ring-doorbell +python-rtmidi +scanless +python-scikit-misc +python-sciunit +python-setuptools_git +python-shade +python-shelly +python-shodan +signon-glib +python-simframe +sipvicious python-sklearn-genetic-opt -python-slip -python-smart-gardena -python-smbpasswd -python-sphinxcontrib-actdiag -python-stdio-mgr -python-streamlink -sword -python-tambo -python-test_server -python-upoints -python-uri-templates -python-uvicorn -python-vcstools -vertica-python -python-visionegg-quest -python-yamlordereddictloader -python-yourls -transmageddon -python3-docs +python-slacker +python-smi +snallygaster +python-snuggs +socialscan +python-socks5line +python-sortedcollections +python-sphinx-documatt-theme +python-sphinx-sitemap +python-sphinxcontrib-asyncio +python-sphinxcontrib-phpdomain +python-sphobjinv +python-sqlmodel +python-ssdp +python-sseclient +python-sseclient-py +python-stackprinter +python-stdiomask +python-stopit +python-subarulink +python-tasmotadevicecontroller +python-tbtrim +python-teslajsonpy +python-testing.common.database +python-textparser +python-textwrap3 +python-toml +python-toposort +python-tosca-parser +python-typedecorator +python-universal-pathlib +python-upnpy +urlbuster +python-uvloop +python-vconnector +python-velbus +strongswan +python-volkszaehler +python-voluptuous-serialize +python-vsure +python-waqiasync +python-waterfurnace +python-webthing +python-webthing-ws +python-whichcraft +python-wiffi +python-winacl +python-winsspi +python-wloc +python-xboxapi +python-xiaomi-gateway +python-xpath-expressions +python-yappi +python-yaswfp +python-zc-customdoctests +python-zm +rapid-photo-downloader +rst2txt +transifex-client +wad +wapiti +webtech +wfuzz +wxGlade +x-tile +xortool +zeek diff --git a/resolve_buildroot.py b/resolve_buildroot.py index dc817d5..847ebbd 100644 --- a/resolve_buildroot.py +++ b/resolve_buildroot.py @@ -10,7 +10,7 @@ # Some deps are only pulled in when those are installed: DEFAULT_GROUPS = ( 'buildsys-build', # for composed repo - #'build', # for koji repo + # 'build', # for koji repo ) diff --git a/utils.py b/utils.py index 58e195f..ecc9a58 100644 --- a/utils.py +++ b/utils.py @@ -1,3 +1,4 @@ +import subprocess import sys import tomllib @@ -26,3 +27,10 @@ def stringify(lst, separator=', '): If no separator is given, separates the items by comma and space. """ return separator.join(name_or_str(i) for i in lst) + + +def run(*cmd, **kwargs): + kwargs.setdefault('check', True) + kwargs.setdefault('capture_output', True) + kwargs.setdefault('text', True) + return subprocess.run(cmd, **kwargs)