From 69d57185dda1477b053883c08565a263fe961e11 Mon Sep 17 00:00:00 2001 From: Brandon Fergerson Date: Wed, 17 Nov 2021 17:29:04 -0500 Subject: [PATCH 1/6] assembleUp --- .github/workflows/e2e.yml | 6 +++--- build.gradle.kts | 24 ++++++++++++++++++++++++ settings.gradle | 5 +++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 200e348..0dd4c3f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -25,9 +25,9 @@ jobs: - run: | PROBE_VERSION="$(python setup.py --version)" echo "SPP_PROBE_VERSION=$PROBE_VERSION" >> $GITHUB_ENV - - run: python setup.py sdist - - run: cp dist/sourceplusplus-${{ env.SPP_PROBE_VERSION }}.tar.gz e2e - - run: cd e2e && docker-compose up -d + + - run: ./gradlew assembleUp + - name: Docker IPs run: docker inspect --format='{{.Name}} - {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -aq) - name: Set E2E_APP_HOST diff --git a/build.gradle.kts b/build.gradle.kts index 0992e05..0a6a715 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,8 +1,32 @@ plugins { id("ru.vyarus.use-python") version "2.3.0" + id("com.avast.gradle.docker-compose") } python { pip("apache-skywalking:0.7.0") pip("vertx-eventbus-client:1.0.0") } + +tasks { + register("buildDist") { + commandLine("sh", "-c", "python setup.py sdist") + } + + register("updateDockerFiles") { + dependsOn("buildDist") + from("dist/") + into("e2e/") + } + + register("assembleUp") { + dependsOn("updateDockerFiles", "composeUp") + } + getByName("composeUp").mustRunAfter("updateDockerFiles") +} + +dockerCompose { + dockerComposeWorkingDirectory.set(File("./e2e")) + removeVolumes.set(true) + waitForTcpPorts.set(false) +} diff --git a/settings.gradle b/settings.gradle index e69de29..87326d1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -0,0 +1,5 @@ +pluginManagement { + plugins { + id 'com.avast.gradle.docker-compose' version "0.14.9" apply false + } +} From 7557927e1ddc6cd7a9ac40018ae16ca932574a3e Mon Sep 17 00:00:00 2001 From: Brandon Fergerson Date: Sat, 20 Nov 2021 13:57:43 -0500 Subject: [PATCH 2/6] load passed args --- sourceplusplus/SourcePlusPlus.py | 43 ++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/sourceplusplus/SourcePlusPlus.py b/sourceplusplus/SourcePlusPlus.py index 16f01c9..cd03b1e 100644 --- a/sourceplusplus/SourcePlusPlus.py +++ b/sourceplusplus/SourcePlusPlus.py @@ -26,17 +26,25 @@ def get_config_value(self, env, default, true_default): else: return true_default - def __init__(self, **kwargs): + def __init__(self, args: dict = None): + if args is None: + args = {} probe_config_file = os.getenv("SPP_PROBE_CONFIG_FILE", "spp-probe.yml") probe_config = {} if os.path.exists(probe_config_file): probe_config = yaml.full_load(open(probe_config_file, "r")) - else: + + # ensure probe_config has required keys + if probe_config.get("spp") is None: probe_config["spp"] = {} + if probe_config.get("skywalking") is None: probe_config["skywalking"] = {} + if probe_config["skywalking"].get("collector") is None: probe_config["skywalking"]["collector"] = {} + if probe_config["skywalking"].get("agent") is None: probe_config["skywalking"]["agent"] = {} + # set default values probe_config["spp"]["probe_id"] = self.get_config_value( "SPP_PROBE_ID", probe_config["spp"].get("probe_id"), str(uuid.uuid4()) ) @@ -46,12 +54,12 @@ def __init__(self, **kwargs): probe_config["spp"]["platform_port"] = self.get_config_value( "SPP_PLATFORM_PORT", probe_config["spp"].get("platform_port"), 5450 ) - probe_config["spp"]["verify_host"] = self.get_config_value( + probe_config["spp"]["verify_host"] = str(self.get_config_value( "SPP_TLS_VERIFY_HOST", probe_config["spp"].get("verify_host"), True - ) - probe_config["spp"]["disable_tls"] = self.get_config_value( + )).lower() == "true" + probe_config["spp"]["disable_tls"] = str(self.get_config_value( "SPP_DISABLE_TLS", probe_config["spp"].get("disable_tls"), False - ) + )).lower() == "true" probe_config["skywalking"]["agent"]["service_name"] = self.get_config_value( "SPP_SERVICE_NAME", probe_config["skywalking"]["agent"].get("service_name"), "spp" ) @@ -63,11 +71,20 @@ def __init__(self, **kwargs): probe_config["skywalking"]["collector"].get("backend_service"), skywalking_host + ":" + str(skywalking_port) ) - self.probe_config = probe_config + for key, val in args.items(): + tmp_config = probe_config + loc = key.split(".") + for i in range(len(loc)): + if tmp_config.get(loc[i]) is None: + tmp_config[loc[i]] = {} + if i == len(loc) - 1: + tmp_config[loc[i]] = val + else: + tmp_config = tmp_config[loc[i]] + + self.probe_config = probe_config self.instrument_remote = None - for key, val in kwargs.items(): - self.__dict__[key] = val def attach(self): config.init( @@ -88,7 +105,13 @@ def attach(self): ssl_ctx = ssl.create_default_context(cadata=ca_data) ssl_ctx.check_hostname = self.probe_config["spp"]["verify_host"] - ssl_ctx.verify_mode = ssl.CERT_NONE # todo: CERT_REQUIRED / load_verify_locations ? + if self.probe_config["spp"]["disable_tls"] is True: + ssl_ctx = None + elif ssl_ctx.check_hostname is True: + ssl_ctx.verify_mode = ssl.CERT_REQUIRED + else: + ssl_ctx.verify_mode = ssl.CERT_NONE + eb = EventBus( host=self.probe_config["spp"]["platform_host"], port=self.probe_config["spp"]["platform_port"], ssl_context=ssl_ctx From 6fd882ed313ac6f44eb7cff11ec3cd12b45dfdc6 Mon Sep 17 00:00:00 2001 From: Brandon Fergerson Date: Sat, 20 Nov 2021 14:00:30 -0500 Subject: [PATCH 3/6] bump --- setup.py | 2 +- sourceplusplus/__init__.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index d0f90f4..a8629f4 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup setup(name='sourceplusplus', - version='0.1.2', + version='0.1.3', description='Source++ Python Probe', url='https://github.com/sourceplusplus/probe-python', author='Source++', diff --git a/sourceplusplus/__init__.py b/sourceplusplus/__init__.py index e2d5bf8..8a3c2fa 100644 --- a/sourceplusplus/__init__.py +++ b/sourceplusplus/__init__.py @@ -1,7 +1,3 @@ -__version__ = '0.1.2' +__version__ = '0.1.3' __name__ = 'Source++' agent_name = 'Source++ Python Probe' - -__version_major__ = '0' -__version_minor__ = '1' -__version_micro__ = '2' From ba4c570b39b1333bdf2dd8abf1f1b2ef3ce0b265 Mon Sep 17 00:00:00 2001 From: Brandon Fergerson Date: Sat, 20 Nov 2021 14:05:10 -0500 Subject: [PATCH 4/6] Update Dockerfile --- e2e/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/Dockerfile b/e2e/Dockerfile index eafe6fa..bd15748 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -6,9 +6,9 @@ RUN pip install Flask RUN pip install PyYAML RUN pip install vertx-eventbus-client -COPY sourceplusplus-0.1.2.tar.gz . +COPY sourceplusplus-0.1.3.tar.gz . -RUN pip install sourceplusplus-0.1.2.tar.gz +RUN pip install sourceplusplus-0.1.3.tar.gz COPY E2ETest.py . From bb0fbcc8b2129bd105eda3de1ff38479a70940b4 Mon Sep 17 00:00:00 2001 From: Brandon Fergerson Date: Sat, 20 Nov 2021 14:10:50 -0500 Subject: [PATCH 5/6] Update docker-compose.yml --- e2e/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index 2f72b93..1ab15b7 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -1,6 +1,7 @@ version: '3.3' services: e2e-test: + container_name: e2e-test build: context: . depends_on: From 52620899cb9bbb613f59a209ae3d9d039273e0dd Mon Sep 17 00:00:00 2001 From: Brandon Fergerson Date: Sat, 20 Nov 2021 14:11:10 -0500 Subject: [PATCH 6/6] Update e2e.yml --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0dd4c3f..4d19749 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -31,7 +31,7 @@ jobs: - name: Docker IPs run: docker inspect --format='{{.Name}} - {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -aq) - name: Set E2E_APP_HOST - run: E2E_APP_HOST=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -aqf "name=e2e_e2e-test_1")) && echo "E2E_APP_HOST=$E2E_APP_HOST" >> $GITHUB_ENV + run: E2E_APP_HOST=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -aqf "name=e2e-test")) && echo "E2E_APP_HOST=$E2E_APP_HOST" >> $GITHUB_ENV - name: Set SPP_PLATFORM_HOST run: SPP_PLATFORM_HOST=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -aqf "name=spp-platform")) && echo "SPP_PLATFORM_HOST=$SPP_PLATFORM_HOST" >> $GITHUB_ENV - name: Wait for platform