Skip to content

Commit c202f64

Browse files
vishwajeet-ibmVishwajeet-Rajopadhye1
vishwajeet-ibm
and
Vishwajeet-Rajopadhye1
authored
GHA Enablement (#46)
* Create pr-build.yml * Update pr-build.yml * Update pr-build.yml * Create validate_builds.py * Update pr-build.yml * Update pr-build.yml * Update pr-build.yml * Update pr-build.yml * Update pr-build.yml * Create currency-build.yaml * Create read_buildinfo.sh * Create build_package.sh * Create pre_process.sh * Create build_wheels.py * Create trivy_code_scan.sh * Create syft_code_scan.sh * Create grype_code_scan.sh * Create build_docker.sh * Create grype_image_scan.sh * Create trivy_image_scan.sh * added gha-scripts * Update validate_builds.py * Update currency-build.yaml * Update build_wheels.sh * Update build_package.sh * Update trivy_code_scan.sh * Update grype_code_scan.sh * Update syft_code_scan.sh * Update grype_image_scan.sh * Update syft_image_scan.sh * Update trivy_image_scan.sh * Update read_buildinfo.sh * Update currency-build.yaml * Update trivy_image_scan.sh * Update trivy_image_scan.sh * Update trivy_image_scan.sh * Update trivy_code_scan.sh * Update trivy_code_scan.sh * Update trivy_image_scan.sh * Update trivy_code_scan.sh * Update trivy_code_scan.sh * Update trivy_code_scan.sh * Update trivy_code_scan.sh * Update trivy_code_scan.sh * Update trivy_code_scan.sh * Update trivy_code_scan.sh * Update upload_file.sh * Update upload_file.sh * Update upload_file.sh * Update upload_file.sh * Update currency-build.yaml * Update currency-build.yaml * Update currency-build.yaml * Update trivy_image_scan.sh * Update trivy_image_scan.sh * Update currency-build.yaml * Update currency-build.yaml * Update trivy_code_scan.sh --------- Co-authored-by: Vishwajeet-Rajopadhye1 <Vishwajeet-Rajopadhye1@ibm.com>
1 parent 9586c35 commit c202f64

26 files changed

+2637
-0
lines changed

‎.github/workflows/currency-build.yaml‎

Lines changed: 418 additions & 0 deletions
Large diffs are not rendered by default.

‎.github/workflows/pr-build.yml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
name: PR build
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- main
7+
- replica-master
8+
9+
jobs:
10+
validate:
11+
runs-on: ubuntu-24.04-s390x # Ensure this self-hosted runner exists
12+
13+
steps:
14+
- name: Checkout code
15+
uses: actions/checkout@v4
16+
17+
- name: Install required packages
18+
run: |
19+
sudo apt update -y
20+
sudo apt install -y software-properties-common
21+
sudo add-apt-repository universe
22+
sudo apt update -y
23+
sudo apt-get install -y file python3-pip
24+
- name: Set up Python venv
25+
run: |
26+
sudo apt update
27+
sudo apt install -y python3-venv python3-full
28+
python3 -m venv venv
29+
source venv/bin/activate
30+
pip install --upgrade pip
31+
pip install requests==2.31.0 docker
32+
# - name: Install Python dependencies
33+
# run: |
34+
# pip3 install --force-reinstall -v "requests==2.31.0"
35+
# pip3 install --upgrade docker
36+
37+
- name: Set PR number
38+
run: echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
39+
40+
- name: Run validate_builds.py script with live logs
41+
run: |
42+
source venv/bin/activate
43+
python3 -u gha-script/validate_builds.py ${PR_NUMBER:-false} 2>&1 | tee build_log
44+
my_pid_status=${PIPESTATUS[0]}
45+
46+
build_size=$(stat -c %s build_log)
47+
48+
if [ "$my_pid_status" -ne 0 ]; then
49+
echo "Script failed for PR #${PR_NUMBER}"
50+
if [ "$build_size" -lt 1800000 ]; then
51+
cat build_log
52+
else
53+
echo "Build log too large, showing last 100 lines"
54+
tail -100 build_log
55+
fi
56+
exit 1
57+
else
58+
echo "Script completed successfully for PR #${PR_NUMBER}"
59+
if [ "$build_size" -lt 1800000 ]; then
60+
cat build_log
61+
else
62+
echo "Build log too large, showing last 100 lines"
63+
tail -100 build_log
64+
fi
65+
fi

‎gha-script/build_docker.sh‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
#!/bin/bash -e
2+
3+
version="$VERSION"
4+
package_dirpath="$PKG_DIR_PATH"
5+
config_file="build_info.json"
6+
image_name=$IMAGE_NAME
7+
build_docker=$BUILD_DOCKER
8+
9+
CUR_DIR=$(pwd)
10+
cd $package_dirpath
11+
12+
# Using python script to find matched version string/key in build_info.json for version passed
13+
match_version=$(python $CUR_DIR/gha-script/match_version_buildinfo.py)
14+
15+
if [ $build_docker != false ];then
16+
if [[ $(jq --arg ver "$match_version" '.[$ver]' $config_file) != null ]]; then
17+
docker_builddir=$(jq -r --arg ver "$match_version" '.[$ver].dir' $config_file)
18+
args=$(jq -r --arg ver "$match_version" '.[$ver].args' $config_file)
19+
patches=$(jq -r --arg ver "$match_version" '.[$ver].patches' $config_file)
20+
# By default send PACKAGE_VERSION argument.
21+
build_args="--build-arg PACKAGE_VERSION=$version"
22+
if [ $args != "null" ]; then
23+
for row in $(echo "$args" | jq -r 'to_entries[] | @base64'); do
24+
key=$(echo "$row" | base64 -d | jq -r '.key')
25+
value=$(echo "$row" | base64 -d | jq -r '.value')
26+
build_args=$(echo $build_args --build-arg $key=$value )
27+
done
28+
fi
29+
if [ $patches != null ]; then
30+
for row in $(echo "$patches" | jq -r 'to_entries[] | @base64'); do
31+
key=$(echo "$row" | base64 -d | jq -r '.key')
32+
value=$(echo "$row" | base64 -d | jq -r '.value')
33+
build_args=$(echo $build_args --build-arg $key=$value )
34+
done
35+
fi
36+
if [[ $(jq --arg ver "$match_version" '.[$ver]' $config_file) != null ]] &&
37+
[[ $(jq -r --arg ver "$match_version" '.[$ver].base_docker_image' $config_file) != null ]]; then
38+
basename=$(jq -r --arg ver "$match_version" '.[$ver].base_docker_image' $config_file)
39+
fi
40+
cmd="$build_args -t $image_name $docker_builddir"
41+
#final_upload_image_link=$(DOCKER_UPLOAD_LINK)/$image_name
42+
docker_file_path="${package_dirpath}/Dockerfiles"
43+
fi
44+
45+
cd Dockerfiles
46+
#echo "Deleting existing docker image"
47+
#docker rmi -f ${image_name}
48+
#docker rmi -f ${basename}
49+
echo "Building docker image"
50+
echo "sudo docker build $build_args -t $image_name $docker_builddir"
51+
echo "*************************************************************************************"
52+
# sudo docker build $build_args -t $image_name $docker_builddir > docker_build.log 2>&1 &
53+
# SCRIPT_PID=$!
54+
# while ps -p $SCRIPT_PID > /dev/null
55+
# do
56+
# echo "$SCRIPT_PID is running"
57+
# sleep 100
58+
# done
59+
# wait $SCRIPT_PID
60+
# my_pid_status=$?
61+
62+
sudo docker build $build_args -t $image_name $docker_builddir 2>&1 | tee docker_build.log
63+
my_pid_status=${PIPESTATUS[0]}
64+
docker_build_size=$(stat -c %s docker_build.log)
65+
66+
if [ $my_pid_status != 0 ];
67+
then
68+
if [ $docker_build_size -lt 1800000 ];
69+
then
70+
cat docker_build.log
71+
else
72+
tail -300 docker_build.log
73+
fi
74+
exit 1
75+
else
76+
if [ $docker_build_size -lt 1800000 ];
77+
then
78+
cat docker_build.log
79+
else
80+
tail -300 docker_build.log
81+
fi
82+
fi
83+
docker save -o image.tar $image_name
84+
else
85+
echo "Docker image is not supported"
86+
fi
87+
88+
# # Publish code keeping commented for now
89+
# if [ $? == 0 ]
90+
# then
91+
# sudo docker tag ${image_name} ${docker_upload_link}/${image_name}
92+
# sudo docker login -u ${env.dockerHubUser} -p ${env.dockerHubPassword} ${docker_upload_link}
93+
# sudo docker push ${docker_upload_link}/${image_name}
94+
# fi

‎gha-script/build_package.sh‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
#!/bin/bash -e
2+
3+
# sudo apt update -y && sudo apt-get install file -y
4+
# #pip3 install --upgrade requests
5+
# pip3 install --force-reinstall -v "requests==2.31.0"
6+
# pip3 install --upgrade docker
7+
sudo apt update -y
8+
sudo apt install -y software-properties-common
9+
sudo add-apt-repository universe -y
10+
sudo apt update -y
11+
sudo apt-get install -y file python3-pip python3-venv python3-full
12+
python3 -m venv venv
13+
source venv/bin/activate
14+
pip install --upgrade pip
15+
pip install requests==2.31.0 docker
16+
17+
echo "Running build script execution in background for "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" "
18+
echo "*************************************************************************************"
19+
20+
docker_image=""
21+
22+
# the below function is used for building a custom docker image, it will be called only when non root user build is set to true.
23+
# function accepts one argument, which is the base image value.
24+
docker_build_non_root() {
25+
echo "building docker image for non root user build"
26+
docker build --build-arg BASE_IMAGE="$1" -t docker_non_root_image -f gha-script/dockerfile_non_root .
27+
docker_image="docker_non_root_image"
28+
}
29+
30+
#Below conditions are used to select the base image based on the 2 flags, tested_on and non_root_build. A docker_build_non_root function is called when non root build is true.
31+
if [[ "$TESTED_ON" == UBI:9* || "$TESTED_ON" == UBI9* ]];
32+
then
33+
ubi_version=$(echo "$TESTED_ON" | grep -oE '[0-9]+\.[0-9]+')
34+
docker pull registry.access.redhat.com/ubi9/ubi:$ubi_version
35+
docker_image="registry.access.redhat.com/ubi9/ubi:$ubi_version"
36+
if [[ "$NON_ROOT_BUILD" == "true" ]];
37+
then
38+
docker_build_non_root "registry.access.redhat.com/ubi9/ubi:$ubi_version"
39+
fi
40+
else
41+
docker pull registry.access.redhat.com/ubi8/ubi:8.7
42+
docker_image="registry.access.redhat.com/ubi8/ubi:8.7"
43+
if [[ "$NON_ROOT_BUILD" == "true" ]];
44+
then
45+
docker_build_non_root "registry.access.redhat.com/ubi8/ubi:8.7"
46+
fi
47+
fi
48+
49+
# python3 script/validate_builds_currency.py "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" "$docker_image" > build_log &
50+
51+
# SCRIPT_PID=$!
52+
# while ps -p $SCRIPT_PID > /dev/null
53+
# do
54+
# echo "$SCRIPT_PID is running"
55+
# sleep 100
56+
# done
57+
# wait $SCRIPT_PID
58+
python3 gha-script/validate_builds_currency.py "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" "$docker_image" 2>&1 | tee build_log
59+
my_pid_status=${PIPESTATUS[0]}
60+
61+
build_size=$(stat -c %s build_log)
62+
63+
if [ $my_pid_status != 0 ];
64+
then
65+
echo "Script execution failed for "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" "
66+
echo "*************************************************************************************"
67+
if [ $build_size -lt 1800000 ];
68+
then
69+
cat build_log
70+
else
71+
tail -100 build_log
72+
fi
73+
exit 1
74+
else
75+
echo "Script execution completed successfully for "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" "
76+
echo "*************************************************************************************"
77+
if [ $build_size -lt 1800000 ];
78+
then
79+
cat build_log
80+
else
81+
tail -100 build_log
82+
fi
83+
fi
84+
exit 0

‎gha-script/build_wheels.py‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import os
2+
import stat
3+
import requests
4+
import sys
5+
import subprocess
6+
import docker
7+
import json
8+
9+
def trigger_build_wheel(wrapper_file, python_version, image_name, file_name, version):
10+
# Docker client setup
11+
client = docker.DockerClient(base_url='unix://var/run/docker.sock')
12+
13+
# Modify permissions for the main script
14+
st1 = os.stat(wrapper_file)
15+
current_dir = os.getcwd()
16+
17+
os.chmod(f"{current_dir}/{wrapper_file}", st1.st_mode | stat.S_IEXEC)
18+
19+
print(current_dir)
20+
print(f"Running script: {wrapper_file}")
21+
print(f"Additional file used by script: {file_name}")
22+
23+
# Extract just the file names
24+
script_name = file_name.split("/")[1]
25+
26+
try:
27+
# Command to run only the main script (which uses the additional file internally)
28+
command = [
29+
"bash",
30+
"-c",
31+
f"cd /home/tester/ && ./{wrapper_file} {python_version} {file_name} {version}"
32+
]
33+
34+
# Run container
35+
container = client.containers.run(
36+
image_name,
37+
command,
38+
network='host',
39+
detach=True,
40+
volumes={current_dir: {'bind': '/home/tester/', 'mode': 'rw'}}, # Mount current directory with both files
41+
stderr=True,
42+
stdout=True
43+
)
44+
45+
# STREAM logs in real-time
46+
for log in container.logs(stream=True, stdout=True, stderr=True, follow=True):
47+
print(log.decode("utf-8").rstrip())
48+
49+
# Wait until it's done
50+
result = container.wait()
51+
52+
except Exception as e:
53+
print(f"Failed to create container: {e}")
54+
raise
55+
56+
finally:
57+
# Clean up container
58+
try:
59+
container.remove()
60+
except:
61+
pass
62+
63+
64+
# Check exit status
65+
if int(result["StatusCode"]) != 0:
66+
raise Exception(f"Build script validation failed for {file_name}!")
67+
else:
68+
return True
69+
70+
if __name__=="__main__":
71+
print("Inside python program")
72+
trigger_build_wheel(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],sys.argv[5])

0 commit comments

Comments
 (0)