diff --git a/bconds.py b/bconds.py index 586904da..58fca91b 100644 --- a/bconds.py +++ b/bconds.py @@ -1,10 +1,12 @@ +import argparse +import datetime import functools +import os import pathlib -import re -import subprocess import sys -from utils import CONFIG, log +from gitrepo import clone_into, refresh_gitrepo, patch_spec, refresh_or_clone +from utils import CONFIG, log, run KOJI_ID_FILENAME = 'koji.id' @@ -35,42 +37,7 @@ def bcond_cache_identifier(component_name, bcond_config, *, branch='', target='' branch = '' identifier = f'{component_name}:{withouts_id}:{withs_id}:{replacements_id}:{branch}:{target}' reverse_id_lookup[identifier] = bcond_config - return identifier - - -def run(*cmd, **kwargs): - kwargs.setdefault('check', True) - kwargs.setdefault('capture_output', True) - kwargs.setdefault('text', True) - return subprocess.run(cmd, **kwargs) - - -def clone_into(component_name, target, branch=''): - branch = branch or CONFIG['distgit']['branch'] - log(f' • Cloning {component_name} into "{target}"...', end=' ') - # I would like to use --depth=1 but that breaks rpmautospec - # https://pagure.io/fedora-infra/rpmautospec/issue/227 - run('fedpkg', 'clone', component_name, target, f'--branch={branch}') - log('done.') - - -def refresh_gitrepo(repopath, prune_exisitng=False): - log(f' • Refreshing "{repopath}" git repo...', end=' ') - git = 'git', '-C', repopath - head_before = run(*git, 'rev-parse', 'HEAD').stdout.rstrip() - run(*git, 'stash') - run(*git, 'reset', '--hard') - run(*git, 'pull') - head_after = run(*git, 'rev-parse', 'HEAD').stdout.rstrip() - if head_before == head_after: - if not prune_exisitng: - # we try to preserve the changes for local inspection, but if it fails, meh - run(*git, 'stash', 'pop', check=False) - log('already up to date.') - return False - else: - log(f'updated {head_before[:10]}..{head_after[:10]}.') - return True + return identifier def srpm_path(directory): @@ -87,28 +54,6 @@ def srpm_path(directory): return candidates[0] -def patch_spec(specpath, bcond_config): - log(f' • Patching {specpath.name}') - - run('git', '-C', specpath.parent, 'reset', '--hard') - - spec_text = specpath.read_text() - - lines = [] - for without in sorted(bcond_config.get('withouts', ())): - if without in bcond_config.get('withs', ()): - raise ValueError(f'Cannot have the same with and without: {without}') - lines.append(f'%global _without_{without} 1') - for with_ in sorted(bcond_config.get('withs', ())): - lines.append(f'%global _with_{with_} 1') - for macro, value in bcond_config.get('replacements', {}).items(): - spec_text = re.sub(fr'^(\s*)%(define|global)(\s+){macro}(\s+)\S.*$', - fr'\1%\2\g<3>{macro}\g<4>{value}', - spec_text, flags=re.MULTILINE) - lines.append(spec_text) - specpath.write_text('\n'.join(lines)) - - def submit_scratchbuild(repopath, target=''): command = ('fedpkg', 'build', '--scratch', '--srpm', f'--arches={CONFIG["architectures"]["koji"]}', '--nowait', '--background') @@ -139,10 +84,24 @@ def koji_status(koji_id): for line in output: if line.startswith('State: '): return line.split(' ')[-1] - raise RuntimeError('Carnot parse koji taskinfo output') + raise RuntimeError('Cannot parse koji taskinfo output') -def handle_exisitng_srpm(repopath, *, was_updated): +def koji_id_is_older_than_week(koji_id_path): + """ + Returns True if koji_id file was created earlier than a week ago. + We assume this is a treshold time after which Koji artifacts are deleted, + and we need to remove koji_id to retrigger the scratchbuild. + If koji_id was created within the last week, return False. + """ + koji_id_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(koji_id_path)) + one_week_ago = datetime.datetime.now() - datetime.timedelta(weeks=1) + if koji_id_mtime < one_week_ago: + return True + return False + + +def handle_existing_srpm(repopath, *, was_updated): srpm = srpm_path(repopath) if srpm and not was_updated: log(f' • Found {srpm.name}, will not rebuild; remove it to force me.') @@ -152,7 +111,7 @@ def handle_exisitng_srpm(repopath, *, was_updated): return None -def handle_exisitng_koji_id(repopath, *, was_updated): +def handle_existing_koji_id(repopath, *, was_updated): koji_id_path = repopath / KOJI_ID_FILENAME if koji_id_path.exists(): if was_updated: @@ -166,13 +125,17 @@ def handle_exisitng_koji_id(repopath, *, was_updated): f'removing {KOJI_ID_FILENAME}.') koji_id_path.unlink() return None + elif status == 'closed' and koji_id_is_older_than_week(koji_id_path): + log(f' • Koji task {koji_task_id} is older than one week, ' + f'there may be nothing to download; removing {KOJI_ID_FILENAME}.') + koji_id_path.unlink() else: log(f' • Koji task {koji_task_id} is {status}; ' - f'not rebulding (rm {KOJI_ID_FILENAME} to force).') + f'not rebuilding (rm {KOJI_ID_FILENAME} to force).') return koji_task_id -def scratchbuild_patched_if_needed(component_name, bcond_config, *, branch='', target=''): +def scratchbuild_patched_if_needed(component_name, bcond_config, *, branch='', target='', no_git_refresh=False): """ This will: 1. clone/fetch the given component_name package from Fedora to fedpkg_cache_dir @@ -187,18 +150,14 @@ def scratchbuild_patched_if_needed(component_name, bcond_config, *, branch='', t 6. return True if something was submitted to Koji """ repopath = pathlib.Path(CONFIG['cache_dir']['fedpkg']) / bcond_config['id'] - if repopath.exists(): - news = refresh_gitrepo(repopath) - else: - pathlib.Path(CONFIG['cache_dir']['fedpkg']).mkdir(exist_ok=True) - clone_into(component_name, repopath, branch=branch) - news = True - if srpm := handle_exisitng_srpm(repopath, was_updated=news): + news = refresh_or_clone(repopath, component_name, no_git_refresh=no_git_refresh, branch=branch) + + if srpm := handle_existing_srpm(repopath, was_updated=news): bcond_config['srpm'] = srpm return False - if koji_id := handle_exisitng_koji_id(repopath, was_updated=news): + if koji_id := handle_existing_koji_id(repopath, was_updated=news): bcond_config['koji_task_id'] = koji_id return False @@ -213,7 +172,7 @@ def scratchbuild_patched_if_needed(component_name, bcond_config, *, branch='', t return True -def download_srpm_if_possible(component_name, bcond_config): +def download_srpm_if_possible(bcond_config): """ This will: 1. inspect the bcond_config for srpm path and a koji build id @@ -230,13 +189,13 @@ def download_srpm_if_possible(component_name, bcond_config): command = ('koji', 'download-task', bcond_config['koji_task_id'], '--arch=src', '--noprogress') koji_output = run(*command, cwd=repopath).stdout.splitlines() if (l := len(koji_output)) != 1: - raise RuntimeError(f'Cannot parse koji download-task ouptut, expected 1 line, got {l}') + raise RuntimeError(f'Cannot parse koji download-task output, expected 1 line, got: {l}') srpm_filename = koji_output[0].split(' ')[-1] if not srpm_filename.endswith('.src.rpm'): - raise RuntimeError('Cannot parse koji download-task ouptut, expected a *.src.rpm filename, got {srpm_filename}') + raise RuntimeError(f'Cannot parse koji download-task output, expected a *.src.rpm filename, got: {srpm_filename}') srpm = repopath / srpm_filename if not srpm.exists(): - raise RuntimeError('Downloaded SRPM does not exist: {srpm}') + raise RuntimeError(f'Downloaded SRPM does not exist: {srpm}') bcond_config['srpm'] = srpm log(srpm_filename) return True @@ -255,7 +214,7 @@ def rpm_requires(rpm): return tuple(sorted({r for r in raw_requires if not r.startswith('rpmlib(')})) -def extract_buildrequires_if_possible(component_name, bcond_config): +def extract_buildrequires_if_possible(bcond_config): """ This will: 1. inspect the bcond_config for srpm path @@ -285,11 +244,31 @@ def build_reverse_id_lookup(): pass +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + '-R', '--no-git-refresh', + action='store_true', + default=False, + help="Don't refresh the gitrepo of each existing component, just send new components scratchbuilds and downloads srpms." + ) + parser.add_argument( + 'packages', + nargs='*', + help='Only fetch bconds for given package name(s).' + ) + return parser.parse_args() + + if __name__ == '__main__': + args = parse_args() + # build everything something_was_submitted = False for component_name, bcond_config in each_bcond_name_config(): - something_was_submitted |= scratchbuild_patched_if_needed(component_name, bcond_config) + if args.packages and component_name not in args.packages: + continue + something_was_submitted |= scratchbuild_patched_if_needed(component_name, bcond_config, no_git_refresh=args.no_git_refresh) # download everything until there's nothing downloaded # the idea is that while downloading, other tasks could finish @@ -299,10 +278,12 @@ def build_reverse_id_lookup(): something_was_downloaded = False # while we were downloading, we could have finished Koji builds for pkg, bcond_configs in CONFIG['bconds'].items(): + if args.packages and pkg not in args.packages: + continue for bcond_config in bcond_configs: if 'buildrequires' not in bcond_config: - something_was_downloaded |= download_srpm_if_possible(component_name, bcond_config) - if extract_buildrequires_if_possible(component_name, bcond_config): + something_was_downloaded |= download_srpm_if_possible(bcond_config) + if extract_buildrequires_if_possible(bcond_config): extracted_count += 1 koji_status.cache_clear() diff --git a/build.py b/build.py index 9d5383cf..804a3195 100644 --- a/build.py +++ b/build.py @@ -1,14 +1,9 @@ import pathlib import sys -# this module reuses bconds functions heavily -# XXX move to a common module? -from bconds import clone_into, refresh_gitrepo, patch_spec, run - -# the following bcond things actually do stay there from bconds import reverse_id_lookup, build_reverse_id_lookup - -from utils import CONFIG +from gitrepo import clone_into, refresh_gitrepo, patch_spec, refresh_or_clone +from utils import CONFIG, run PATCHDIR = pathlib.Path('patches_dir') @@ -26,13 +21,8 @@ bootstrap = reverse_id_lookup[component_name] component_name, *_ = component_name.partition(':') - # XXX make a reusable function with just refresh_gitrepo/clone_into repopath = FEDPKG_CACHEDIR / component_name - if repopath.exists(): - refresh_gitrepo(repopath, prune_exisitng=True) - else: - FEDPKG_CACHEDIR.mkdir(exist_ok=True) - clone_into(component_name, repopath) + refresh_or_clone(repopath, component_name, prune_existing=True) specpath = repopath / f'{component_name}.spec' @@ -55,19 +45,19 @@ # Bump and commit only if we haven't already, XXX ability to force this head_commit_msg = run('git', '-C', repopath, 'log', '--format=%B', '-n1', 'HEAD').stdout.rstrip() - if False: # and bootstrap or head_commit_msg != message: + if bootstrap: # and bootstrap or head_commit_msg != message: run('rpmdev-bumpspec', '-c', message, '--userstring', CONFIG['distgit']['author'], specpath) run('git', '-C', repopath, 'commit', '--allow-empty', f'{component_name}.spec', '-m', message, '--author', CONFIG['distgit']['author']) - #raise NotImplementedError('no pushing yet') + # raise NotImplementedError('no pushing yet') run('git', '-C', repopath, 'push') - run('fedpkg', 'build', '--fail-fast', '--nowait', '--target', CONFIG['koji']['target'], cwd=repopath) # '--background' + run('fedpkg', 'build', '--fail-fast', '--nowait', '--background', '--target', CONFIG['koji']['target'], cwd=repopath) - # XXX prune this directory becasue we don't want no thousands clones? + # XXX prune this directory because we don't want no thousands clones? # maybe we are not gonna need this? except Exception: print(sys.argv[1]) raise - # XXX prune this directory becasue we don't want no thousands clones? + # XXX prune this directory because we don't want no thousands clones? # maybe we are not gonna need this? diff --git a/config.toml b/config.toml index dc3cc480..5f82323a 100644 --- a/config.toml +++ b/config.toml @@ -4,36 +4,31 @@ ## build python3.N-1 together with python3.N without any bcond, verify python3.N-1 is nonmain python #- python3.N: # macros: -# _with_bootstrap: 1 # bump the release so it's greater even with ~bootstrap -# _without_rpmwheels: 1 -# _without_tests: 1 -# _without_optimizations: 1 +# %global _with_bootstrap 1 # bump the release so it's greater even with ~bootstrap +# %global _without_rpmwheels 1 +# %global _without_tests 1 +# %global _without_optimizations 1 #- gdb: # not blocking (yet) # macros: -# _without_python: 1 +# %global _without_python 1 #- python-flit-core: # macros: -# _with_bootstrap: 1 # bump the release so it's greater even with ~bootstrap +# %global _with_bootstrap 1 # bump the release so it's greater even with ~bootstrap #- python-packaging: # macros: -# _with_bootstrap: 1 # bump the release so it's greater even with ~bootstrap +# %global _with_bootstrap 1 # bump the release so it's greater even with ~bootstrap #- python-setuptools: # macros: -# _with_bootstrap: 1 # bump the release so it's greater even with ~bootstrap -#- python-wheel: -# macros: -# _with_bootstrap: 1 # bump the release so it's greater even with ~bootstrap +# %global _with_bootstrap 1 # bump the release so it's greater even with ~bootstrap #- python-pip: # macros: -# _without_tests: 1 -# _without_doc: 1 -#- python-setuptools: -# macros: -# _without_tests: 1 +# %global _without_tests 1 +# %global _without_man 1 #- pyparsing: # python BuildRequires systemtap-sdt-devel Requires pyparsing # macros: -# _without_tests: 1 -# _without_doc: 1 +# %global _without_tests 1 +# %global _without_doc 1 +# %global _without_extras 1 ## wait for gdb #- python3.N # VERIFY all of the above have correct Provides/Requires @@ -41,11 +36,12 @@ [koji] target = 'rawhide' +# target = 'f41-python' [distgit] branch = "rawhide" -commit_message = "Rebuilt for Python 3.12" -bootstrap_commit_message = "Bootstrap for Python 3.12" +commit_message = "Rebuilt for Python 3.14" +bootstrap_commit_message = "Bootstrap for Python 3.14" author = "Python Maint " [architectures] @@ -53,11 +49,11 @@ repoquery = "x86_64" # used for repository querying koji = "x86_64" # used to scratch build packages with bconds [deps] -old = ["python(abi) = 3.11", "libpython3.11.so.1.0()(64bit)", "libpython3.11d.so.1.0()(64bit)"] -new = ["python(abi) = 3.12", "libpython3.12.so.1.0()(64bit)", "libpython3.12d.so.1.0()(64bit)"] +old = ["python(abi) = 3.13", "libpython3.13.so.1.0()(64bit)", "libpython3.13d.so.1.0()(64bit)"] +new = ["python(abi) = 3.14", "libpython3.14.so.1.0()(64bit)", "libpython3.14d.so.1.0()(64bit)"] [components] -excluded = ["python3.11", "python3.12"] +excluded = ["python3.13", "python3.14"] extra = ["python3-docs"] [cache_dir] @@ -68,25 +64,23 @@ fedpkg = "_fedpkg_cache_dir" [[repos.rawhide]] repoid = "rawhide" # metalink = "https://mirrors.fedoraproject.org/metalink?repo=rawhide&arch=$basearch" -baseurl = ["https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20230705.n.0/compose/Everything/$basearch/os/"] -metadata_expire = 600000000 +baseurl = ["http://kojipkgs.fedoraproject.org/repos/rawhide/latest/$basearch/"] [[repos.rawhide]] repoid = "rawhide-source" # metalink = "https://mirrors.fedoraproject.org/metalink?repo=rawhide-source&arch=$basearch" -baseurl = ["https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20230705.n.0/compose/Everything/source/tree/"] -metadata_expire = 600000000 +baseurl = ["http://kojipkgs.fedoraproject.org/repos/rawhide/latest/src/"] + [[repos.target]] -repoid = "python3.12" -#baseurl = ["https://copr-be.cloud.fedoraproject.org/results/@python/python3.12/fedora-rawhide-$basearch/"] -baseurl = ["http://kojipkgs.fedoraproject.org/repos/f39-build/latest/$basearch/"] +repoid = "python3.14" +baseurl = ["https://copr-be.cloud.fedoraproject.org/results/@python/python3.14-b1/fedora-rawhide-$basearch/"] +# baseurl = ["http://kojipkgs.fedoraproject.org/repos/f41-build/latest/$basearch/"] metadata_expire = 60 [bconds] [[bconds.python-setuptools]] withs = ["bootstrap"] -withouts = ["tests"] [[bconds.python-packaging]] withs = ["bootstrap"] @@ -95,7 +89,7 @@ withs = ["bootstrap"] withs = ["bootstrap"] [[bconds.python-pip]] -withouts = ["tests", "doc"] +withouts = ["tests", "man"] [[bconds.python-six]] withouts = ["tests"] @@ -110,7 +104,7 @@ withouts = ["tests"] withouts = ["docs", "tests"] [[bconds.python-chardet]] -withouts = ["doc_pdf"] +withouts = ["doc"] [[bconds.python-pbr]] withs = ["bootstrap"] @@ -171,13 +165,13 @@ withouts = ["tests"] withouts = ["timeout", "tests", "docs"] [[bconds.python-virtualenv]] -withouts = ["tests"] +withs = ["bootstrap"] [[bconds.babel]] withs = ["bootstrap"] [[bconds.python-jinja2]] -withouts = ["docs"] +withouts = ["docs", "asyncio_tests"] [[bconds.python-sphinx_rtd_theme]] withs = ["bootstrap"] @@ -186,31 +180,13 @@ withs = ["bootstrap"] withs = ["bootstrap"] [[bconds.python-urllib3]] -withouts = ["tests"] +withouts = ["tests", "extradeps"] [[bconds.python-requests]] withouts = ["tests"] -[[bconds.python-sphinxcontrib-applehelp]] -withouts = ["check"] - -[[bconds.python-sphinxcontrib-devhelp]] -withouts = ["check"] - -[[bconds.python-sphinxcontrib-htmlhelp]] -withouts = ["check"] - -[[bconds.python-sphinxcontrib-jsmath]] -withouts = ["check"] - -[[bconds.python-sphinxcontrib-qthelp]] -withouts = ["check"] - -[[bconds.python-sphinxcontrib-serializinghtml]] -withouts = ["check"] - [[bconds.python-sphinx]] -withouts = ["tests", "websupport"] +withouts = ["tests", "sphinxcontrib"] [[bconds.python-jedi]] withouts = ["tests"] @@ -227,9 +203,6 @@ withouts = ["optional_tests"] [[bconds.python-soupsieve]] withouts = ["tests"] -[[bconds.python-towncrier]] -withouts = ["tests"] - [[bconds.python-pytest-asyncio]] withouts = ["tests"] @@ -242,17 +215,8 @@ withouts = ["tests"] [[bconds.python-async-timeout]] withouts = ["tests"] -[[bconds.python-trio]] -withouts = ["tests"] - -[[bconds.python-Automat]] -withouts = ["tests"] - -[[bconds.python-invoke]] -withouts = ["tests"] - [[bconds.python-jupyter-client]] -withouts = ["doc","tests"] +withouts = ["tests"] [[bconds.python-jupyter-server]] withouts = ["tests"] @@ -264,7 +228,7 @@ withouts = ["check"] withouts = ["check","doc"] [[bconds.python-ipykernel]] -withouts = ["tests","doc"] +withouts = ["tests"] [[bconds.pybind11]] withouts = ["tests"] @@ -284,36 +248,24 @@ withouts = ["tests"] [[bconds.freeipa-healthcheck]] withouts = ["tests"] -[[bconds.python-zbase32]] -withs = ["bootstrap"] - [[bconds.python-Traits]] withs = ["bootstrap"] -[[bconds.python-lit]] -withouts = ["check"] - [[bconds.python-pcodedmp]] withs = ["bootstrap"] [[bconds.python-libcst]] -withouts = ["tests", "docs"] +withs = ["bootstrap"] [[bconds.python-databases]] withs = ["bootstrap"] -[[bconds.python-molecule]] -withouts = ["doc"] - [[bconds.scipy]] withouts = ["pythran"] [[bconds.python-pandas]] withs = ["bootstrap"] -[[bconds.grpc]] -withs = ["bootstrap"] - [[bconds.python-fasteners]] withouts = ["tests"] @@ -322,7 +274,7 @@ withs = ["bootstrap"] withouts = ["docs"] [[bconds.python-zope-interface]] -withouts = ["tests", "docs"] +withouts = ["docs"] [[bconds.python-tqdm]] withouts = ["tests"] @@ -334,7 +286,7 @@ withouts = ["tests"] withouts = ["tests"] [[bconds.python-geopandas]] -withouts = ["tests"] +withs = ["bootstrap"] [[bconds.python-astropy]] withouts = ["check"] @@ -345,14 +297,13 @@ withouts = ["tests"] [[bconds.python-aiohttp]] withouts = ["tests"] -[[bconds.python-pep517]] -withouts = ["tests"] - [[bconds.python-azure-core]] withouts = ["tests"] -[[bconds.python-azure-common]] -withouts = ["tests"] +# this bcond is disabled in spec on purpose +# it might be enabled in the future, so leaving this here, commend out +#[[bconds.python-azure-common]] +#withouts = ["tests"] [[bconds.python-azure-mgmt-core]] withouts = ["tests"] @@ -361,10 +312,10 @@ withouts = ["tests"] withouts = ["tests"] [[bconds.python-typeguard]] -withouts = ["doc_pdf"] +withs = ["bootstrap"] [[bconds.python-tox]] -withouts = ["tests"] +withs = ["bootstrap"] [[bconds.python-pytest-rerunfailures]] withouts = ["tests"] @@ -381,9 +332,6 @@ withs = ["bootstrap"] [[bconds.python-msrest]] withouts = ["tests"] -[[bconds.numpy]] -withouts = ["tests"] - [[bconds.python-oletools]] withs = ["bootstrap"] @@ -391,8 +339,17 @@ withs = ["bootstrap"] withouts = ["tests"] [[bconds.fonttools]] +withouts = ["tests", "plot_extra", "symfont_extra", "ufo_extra", "woff_extra", "graphite_extra", "interpolatable_extra"] + +[[bconds.conda]] +withouts = ["tests"] + +[[bconds.python-conda-libmamba-solver]] withouts = ["tests"] +[[bconds.python-conda-index]] +withs = ["bootstrap"] + [[bconds.python-conda-package-streaming]] withs = ["bootstrap"] @@ -405,12 +362,6 @@ withouts = ["tests"] [[bconds.python-dns]] withouts = ["trio", "curio", "doh"] -[[bconds.python-poetry-plugin-export]] -withouts = ["bootstrap"] - -[[bconds.poetry]] -withs = ["bootstrap"] - [[bconds.python-networkx]] withs = ["bootstrap"] @@ -421,10 +372,13 @@ withouts = ["soupsieve", "tests"] withouts = ["extras"] [[bconds.python-constantly]] -withouts = ["tests"] +withs = ["bootstrap"] -[[bconds.gdb]] -replacements = {_without_python = "1"} +# this is intentionally commented out +# 1) is is part of the initial bootstrap +# 2) replacement does not work, the macro isn't there (needs to be added) +#[[bconds.gdb]] +#replacements = {_without_python = "1"} [[bconds.python-pysocks]] replacements = {with_python3_tests = "0"} @@ -448,10 +402,178 @@ withouts = ["plugins"] withouts = ["tests"] [[bconds.python-tox-current-env]] -withouts = ["tests"] +withs = ["bootstrap"] [[bconds.python-pikepdf]] -withs = ["docs", "tests"] +withouts = ["docs", "tests"] + +[[bconds.python-executing]] +withs = ["bootstrap"] + +[[bconds.python-editables]] +withouts = ["doc"] + +[[bconds.python-jsonschema-specifications]] +replacements = {with_doc = "0"} + +[[bconds.python-openstackclient]] +replacements = {with_doc = "0"} + +[[bconds.python-oslo-config]] +replacements = {repo_bootstrap = "1"} + +[[bconds.python-neutronclient]] +replacements = {with_doc = "0"} + +[[bconds.python-glanceclient]] +replacements = {with_doc = "0"} + +[[bconds.python-domdf-python-tools]] +withouts = ["tests"] + +[[bconds.python-tiny-proxy]] +withouts = ["tests"] + +[[bconds.python-werkzeug]] +withouts = ["tests"] + +[[bconds.python-netaddr]] +withouts = ["docs"] + +[[bconds.python-dirty-equals]] +withs = ["bootstrap"] + +[[bconds.python-snakemake-interface-storage-plugins]] +withs = ["bootstrap"] + +[[bconds.python-snakemake-interface-executor-plugins]] +withs = ["bootstrap"] + +[[bconds.python-snakemake-interface-report-plugins]] +withs = ["bootstrap"] + +[[bconds.snakemake]] +withs = ["bootstrap"] + +[[bconds.python-sphinx-theme-builder]] +withs = ["bootstrap"] + +[[bconds.python-httpx]] +withouts = ["tests"] + +[[bconds.python-oslo-i18n]] +replacements = {with_doc = "0"} + +[[bconds.python-pymssql]] +withouts = ["tests"] + +[[bconds.python-authlib]] +withouts = ["tests"] + +[[bconds.python-oslotest]] +withs = ["repo_bootstrap"] + +[[bconds.python-gunicorn]] +withouts = ["extras"] + +[[bconds.python-execnet]] +withouts = ["optional_test_deps"] + +[[bconds.python-threadpoolctl]] +withouts = ["check"] + +[[bconds.python-tabulate]] +withs = ["bootstrap"] + +[[bconds.python-repoze-sphinx-autointerface]] +withouts = ["tests"] + +[[bconds.python-zope-exceptions]] +withouts = ["tests"] + +[[bconds.python-deepdiff]] +withouts = ["tests"] + +[[bconds.python-sentry-sdk]] +withouts = ["tests"] + +[[bconds.python-dask]] +withs = ["bootstrap"] + +[[bconds.python-build]] +withouts = ["uv"] + +[[bconds.meson]] +withouts = ["check"] + +[[bconds.python-gmpy2]] +withouts = ["tests"] + +[[bconds.python-humanfriendly]] +withs = ["bootstrap"] + +[[bconds.python-cascadio]] +withs = ["bootstrap"] + +[[bconds.python-pint]] +withouts = ["xarray"] + +[[bconds.python-qcengine]] +withouts = ["tests"] + +[[bconds.python-astropy-iers-data]] +withouts = ["tests"] + +[[bconds.python-pymongo]] +withs = ["bootstrap"] + +[[bconds.python-pyogrio]] +withs = ["bootstrap"] + +[[bconds.python-fastapi]] +withs = ["bootstrap"] + +[[bconds.python-pydantic-core]] +withouts = ["inline_snapshot_tests"] + +[[bconds.python-optking]] +withouts = ["tests"] + +[[bconds.python-django5]] +withouts = ["tests"] + +[[bconds."python-django4.2"]] +withouts = ["tests"] + +[[bconds.python-pyproject-hooks]] +withouts = ["tests"] + +[[bconds.python-Automat]] +withouts = ["doc"] + +[[bconds.python-psutil]] +withouts = ["xdist"] + +[[bconds.python-apkinspector]] +withs = ["bootstrap"] + +[[bconds.python-androguard]] +withs = ["bootstrap"] + +[[bconds.python-service-identity]] +withouts = ["docs"] + +[[bconds.mkdocs-material]] +withs = ["bootstrap"] + +[[bconds.python-mkdocs-material-extensions]] +withouts = ["tests"] + +[[bconds.python-zope-testing]] +withouts = ["tests"] + +[[bconds.python-xnat]] +withouts = ["tests"] # [[bconds.OpenColorIO]] # %global bootstrap 1 diff --git a/gitrepo.py b/gitrepo.py new file mode 100644 index 00000000..ec9580e7 --- /dev/null +++ b/gitrepo.py @@ -0,0 +1,76 @@ +""" +This module contains helper functions to manipulate with distgit repositories: +clone them, refresh their local copies and patch the specfiles. +""" + +import re + +from utils import CONFIG, log, run + + +def clone_into(component_name, target, branch=''): + branch = branch or CONFIG['distgit']['branch'] + log(f' • Cloning {component_name} into "{target}"...', end=' ') + # I would like to use --depth=1 but that breaks rpmautospec + # https://pagure.io/fedora-infra/rpmautospec/issue/227 + run('fedpkg', 'clone', component_name, target, f'--branch={branch}') + log('done.') + + +def refresh_gitrepo(repopath, prune_existing=False): + log(f' • Refreshing "{repopath}" git repo...', end=' ') + git = 'git', '-C', repopath + head_before = run(*git, 'rev-parse', 'HEAD').stdout.rstrip() + run(*git, 'stash') + run(*git, 'reset', '--hard') + run(*git, 'pull') + head_after = run(*git, 'rev-parse', 'HEAD').stdout.rstrip() + if head_before == head_after: + if not prune_existing: + # we try to preserve the changes for local inspection, but if it fails, meh + run(*git, 'stash', 'pop', check=False) + log('already up to date.') + return False + else: + log(f'updated {head_before[:10]}..{head_after[:10]}.') + return True + + +def patch_spec(specpath, bcond_config): + log(f' • Patching {specpath.name}') + + run('git', '-C', specpath.parent, 'reset', '--hard') + + spec_text = specpath.read_text() + + lines = [] + for without in sorted(bcond_config.get('withouts', ())): + if without in bcond_config.get('withs', ()): + raise ValueError(f'Cannot have the same with and without: {without}') + lines.append(f'%global _without_{without} 1') + for with_ in sorted(bcond_config.get('withs', ())): + lines.append(f'%global _with_{with_} 1') + for macro, value in bcond_config.get('replacements', {}).items(): + spec_text = re.sub(fr'^(\s*)%(define|global)(\s+){macro}(\s+)\S.*$', + fr'\1%\2\g<3>{macro}\g<4>{value}', + spec_text, flags=re.MULTILINE) + lines.append(spec_text) + specpath.write_text('\n'.join(lines)) + + +def refresh_or_clone(repopath, component_name, *, prune_existing=False, no_git_refresh=False, branch=''): + """ + Returns True if there's new contents of the repository. + Returns False if the content of the repository remains the same + or if no_git_refresh option is set to True + (we skip the repository update and assume nothing has changed). + """ + if repopath.exists(): + if no_git_refresh: + return False + else: + return refresh_gitrepo(repopath, prune_existing=prune_existing) + else: + repopath.parent.mkdir(exist_ok=True) + clone_into(component_name, repopath, branch=branch) + return True diff --git a/jobs.py b/jobs.py index 2782d5a8..64705f9a 100644 --- a/jobs.py +++ b/jobs.py @@ -1,5 +1,6 @@ import collections import functools +import os import sys from sacks import MULTILIB, rawhide_sack, target_sack @@ -124,14 +125,20 @@ def are_all_done(*, packages_to_check, all_components, components_done, blocker_ # Hence, we only compare the names # For Copr rebuilds, the Copr EVR must be >= Fedora EVR # For koji rebuilds, this will be always true anyway - # XXX cython was renamed after the rebuild - if done_package.name == required_package.name or required_package.name == 'python3-Cython': - #if not done_package.evr_lt(required_package): + if done_package.name == required_package.name: + # if not done_package.evr_lt(required_package): if True: log(f' ✔ {required_package.name}') break else: has_older = True + else: + # Check the virtual provides - maybe one of them matches what we look for + for provide in done_package.provides: + if provide.name == required_package.name: + log(f' ✔ {required_package.name}') + break + break else: if has_older: log(f' ✗ {required_package.name} (older EVR available)') @@ -157,6 +164,7 @@ def _sort_loop(loop): def _detect_loop(loop_detector, probed_component, depchain, loops, seen): for component in loop_detector[probed_component]: + recursedown = component not in seen seen.add(component) if component in CONFIG['bconds']: # we assume bconds are manually crafted not to have loops @@ -166,7 +174,8 @@ def _detect_loop(loop_detector, probed_component, depchain, loops, seen): if component in depchain: loops.add(_sort_loop(depchain[depchain.index(component):])) continue - _detect_loop(loop_detector, component, depchain + [component], loops, seen) + if recursedown: + _detect_loop(loop_detector, component, depchain + [component], loops, seen) def report_blocking_components(loop_detector): loops = set() @@ -174,6 +183,7 @@ def report_blocking_components(loop_detector): for component in loop_detector: if component not in seen: _detect_loop(loop_detector, component, [component], loops, seen) + seen.add(component) log('\nDetected dependency loops:') for loop in sorted(loops, key=lambda t: -len(t)): log(' • ' + ' → '.join(loop)) @@ -204,32 +214,38 @@ def report_blocking_components(loop_detector): component_buildroot = resolve_buildrequires_of(component) except ValueError as e: log(f'\n ✗ {e}') - continue + number_of_resolved = None + ready_to_rebuild = False + else: + number_of_resolved = len(component_buildroot) - ready_to_rebuild = are_all_done( - packages_to_check=set(component_buildroot) & binary_rpms, - all_components=components, - components_done=components_done, - blocker_counter=blocker_counter, - loop_detector=loop_detector, - ) + ready_to_rebuild = are_all_done( + packages_to_check=set(component_buildroot) & binary_rpms, + all_components=components, + components_done=components_done, + blocker_counter=blocker_counter, + loop_detector=loop_detector, + ) if ready_to_rebuild: - # XXX make this configurable - if component not in components_done: + if os.environ.get('PRINT_ALL') or component not in components_done: print(component) elif component in CONFIG['bconds']: for bcond_config in CONFIG['bconds'][component]: bcond_config['id'] = bcond_cache_identifier(component, bcond_config) log(f'• {component} not ready and {bcond_config["id"]} bcond found, will check that one') if 'buildrequires' not in bcond_config: - extract_buildrequires_if_possible(component, bcond_config) + extract_buildrequires_if_possible(bcond_config) if 'buildrequires' in bcond_config: try: component_buildroot = resolve_requires(tuple(sorted(bcond_config['buildrequires']))) except ValueError as e: log(f'\n ✗ {e}') continue + if number_of_resolved == len(component_buildroot): + # XXX when this happens, the bcond might be bogus + # figure out a way to present that information + pass ready_to_rebuild = are_all_done( packages_to_check=set(component_buildroot) & binary_rpms, all_components=components, @@ -238,7 +254,7 @@ def report_blocking_components(loop_detector): loop_detector=loop_detector, ) if ready_to_rebuild: - if component not in components_done: + if os.environ.get('PRINT_ALL') or component not in components_done: print(bcond_config['id']) else: log(f' • {bcond_config["id"]} bcond SRPM not present yet, skipping') diff --git a/progress.pkgs b/progress.pkgs index a4359833..f51f1f01 100644 --- a/progress.pkgs +++ b/progress.pkgs @@ -1,107 +1,165 @@ -andriller -androguard -androwarn -awscli -brd -btest -condor -cvc4 -swid-tools -eric -fail2ban -fedora-gather-easyfix -imgbased -python-mathics3 -pico-wizard -pyflowtools -python-acora -python-adext -python-aioambient -python-aioguardian -python-aiozeroconf -python-airthings -python-alarmdecoder -python-ansi -python-ansible-pygments -python-async-generator -awake -python-blockdiag -python-bluepy -python-box -python-certbot-dns-cloudxns -python-click-spinner -python-cypari2 -python-cypy -dionaea -python-django-contact-form -python-django-robots -python-django3 -python-dominate -python-editdistance-s -python-flask-bootstrap -python-fpylll -python-gccinvocation -python-geomet -python-grako -python-graphitesend -python-igor -python-ipgetter -python-jep -python-jsonrpc-server -python-lacrosse -python-lark-parser -python-lasagne -python-lazr-smtptest -lensfun -python-leveldb -python-liblarch -libssh2-python -python-logging-tree -python-logutils -python-mpd2 -mraa -python-networkmanager -python-nipy -python-nose_fixes -python-notario -openshadinglanguage -python-optcomplete -python-podman-api -python-pvc -python-py9p -python-pyaes -python-pybv -python-pydiffx -pyftpdlib -libgpuarray -python-pymc3 -python-pyngus -python-pyoptical -python-pyside2 -python-pytelegrambotapi -python-pytest-flake8 -python-pytest-metadata -python-pytest-virtualenv -python3-script -python-simpleparse -python-simplewrap -python-sklearn-genetic-opt -python-slip -python-smart-gardena -python-smbpasswd -python-sphinxcontrib-actdiag -python-stdio-mgr -python-streamlink -sword -python-tambo -python-test_server -python-upoints -python-uri-templates -python-uvicorn -python-vcstools -vertica-python -python-visionegg-quest -python-yamlordereddictloader -python-yourls -transmageddon +ansible-core +asv +avogadro2-libs +awscli2 +binwalk +borgbackup +chirp +copr-backend +criu +diceware +did +git-filter-repo +gnofract4d +python-howdoi +httpie +ikiwiki +jrnl +kmymoney +kvirc +libpeas +linode-cli +monkeytype +mopidy +mrchem +ocrmypdf +pcs +poezio +pysubnettree +python-Mastodon +python-aioesphomeapi +python-aiokafka +python-aiosmtpd +python-astdoc +python-avocado +python-awesomeversion +python-axolotl +python-beancount +python-beartype +python-billiard +python-boutdata +python-pymongo +capstone +python-capturer +python-carbon +python-catkin_tools +python-cerberus +python-chalice +pychromecast +cjdns +python-colcon-parallel-executor +python-colorful +python-confuse +python-crcmod +python-crick +python-dataclassy +python-diff-cover +root +python-django-configurations +python-django-contrib-comments +python-django-filter +dlib +python-dunamai +python-envisage +python-eth-stdlib +python-exdir +python3-exiv2 +python-flake8-comprehensions +python-flask-session +python-fs +python-fsleyes-widgets +python-googleapis-common-protos::bootstrap::: +python-gphoto2 +python-h5netcdf +python-hl7 +python-imbalanced-learn +python-inject +python-injector +python-invoke +python-ipfshttpclient +python-irc +python-josepy +qtile +python-loguru +python-makefun +python-markups +Mayavi +python-mechanize +python-mplcursors +python-msal +python-neatdend +python-niaarmts +python-octave-kernel +onnx +opencv +python-osc-lib +python-oslo-log +python-packvers +python-paginate +python-pamela +patool +petsc +python-pika +python-pkginfo2 +python-praw +python-priority +python-pudb +python-pure-protobuf +python-pycec +python-pydantic-core:inline_snapshot_tests:::: +python-pydocstyle +python-pyemd +pylibacl +pylint +python-pymdown-extensions +python-pymeeus +python-pyopencl +python-pyopengl +pyproj +pyserial-asyncio +python-pytesseract +python-pytest-twisted +python-pyupgrade +python-pyvlx +python-rasterio +python-rdflib +python-reactivex +python-readability-lxml +python-recordclass +python-remoto +python-rosdep +python-rq +python-rstcheck +python-rx +python-scikit-image +signon-glib +python-simframe +python-sqlalchemy1.4 +python-stdlibs +subversion +python-telnetlib3 +python-tenacity +python-textual +python-toolz +python-torch +python-towncrier +python-txaio +python-typeguard +python-typing-inspect +python-uvloop +uwsgi +python-valkey +visidata +python-watchfiles +weasyprint +python-webob +python-wordcloud +python-yappi +python-zict +rebase-helper +rpm-head-signing +ufw +xpra +yt-dlp +zeek python3-docs diff --git a/resolve_buildroot.py b/resolve_buildroot.py index dc817d55..755d426b 100644 --- a/resolve_buildroot.py +++ b/resolve_buildroot.py @@ -9,8 +9,8 @@ # Some deps are only pulled in when those are installed: DEFAULT_GROUPS = ( - 'buildsys-build', # for composed repo - #'build', # for koji repo + # 'buildsys-build', # for composed repo + 'build', # for koji repo ) diff --git a/utils.py b/utils.py index 58e195fd..ecc9a582 100644 --- a/utils.py +++ b/utils.py @@ -1,3 +1,4 @@ +import subprocess import sys import tomllib @@ -26,3 +27,10 @@ def stringify(lst, separator=', '): If no separator is given, separates the items by comma and space. """ return separator.join(name_or_str(i) for i in lst) + + +def run(*cmd, **kwargs): + kwargs.setdefault('check', True) + kwargs.setdefault('capture_output', True) + kwargs.setdefault('text', True) + return subprocess.run(cmd, **kwargs)