diff --git a/.github/workflows/git-secrets.yml b/.github/workflows/git-secrets.yml index a3dce06c..cb7dc82e 100644 --- a/.github/workflows/git-secrets.yml +++ b/.github/workflows/git-secrets.yml @@ -9,7 +9,7 @@ jobs: # This workflow contains a single job called "main" git-secrets: # The type of runner that the job will run on - runs-on: ubuntu-18.04 + runs-on: macos-latest # Steps represent a sequence of tasks that will be executed as part of the job steps: @@ -22,7 +22,7 @@ jobs: python-version: 3.8 - name: Installing dependencies run: - sudo apt-get install git less openssh-server + brew install git less openssh - name: Installing scanning tool run: | brew install git-secrets diff --git a/sawmill-core/src/main/java/io/logz/sawmill/PipelineExecutionMetricsMBean.java b/sawmill-core/src/main/java/io/logz/sawmill/PipelineExecutionMetricsMBean.java index e63ad103..749a46e6 100644 --- a/sawmill-core/src/main/java/io/logz/sawmill/PipelineExecutionMetricsMBean.java +++ b/sawmill-core/src/main/java/io/logz/sawmill/PipelineExecutionMetricsMBean.java @@ -76,7 +76,7 @@ public void pipelineFinishedSuccessfully(String pipelineId, Doc doc, long timeTo } @Override - public void processorFailed(String pipelineId, String processorName, Doc doc, Optional optionalError) { + public void processorFailed(String pipelineId, String processorName, Doc doc) { processorsMetrics.computeIfAbsent(processorName, k -> new ProcessorMetrics()).incrementFailure(); } diff --git a/sawmill-core/src/main/java/io/logz/sawmill/PipelineExecutionMetricsTracker.java b/sawmill-core/src/main/java/io/logz/sawmill/PipelineExecutionMetricsTracker.java index 3d59a2d9..68667a35 100644 --- a/sawmill-core/src/main/java/io/logz/sawmill/PipelineExecutionMetricsTracker.java +++ b/sawmill-core/src/main/java/io/logz/sawmill/PipelineExecutionMetricsTracker.java @@ -1,6 +1,5 @@ package io.logz.sawmill; -import java.util.Optional; public interface PipelineExecutionMetricsTracker { void pipelineFinishedSuccessfully(String pipelineId, Doc doc, long timeTookNs); @@ -13,7 +12,7 @@ public interface PipelineExecutionMetricsTracker { void processorFinishedSuccessfully(String pipelineId, String processorName, long timeTookNs); - void processorFailed(String pipelineId, String processorName, Doc doc, Optional optionalError); + void processorFailed(String pipelineId, String processorName, Doc doc); void pipelineFailedOnUnexpectedError(String pipelineId, Doc doc, Exception e); diff --git a/sawmill-core/src/main/java/io/logz/sawmill/PipelineExecutor.java b/sawmill-core/src/main/java/io/logz/sawmill/PipelineExecutor.java index 58ff2ec3..aa480a41 100644 --- a/sawmill-core/src/main/java/io/logz/sawmill/PipelineExecutor.java +++ b/sawmill-core/src/main/java/io/logz/sawmill/PipelineExecutor.java @@ -132,7 +132,7 @@ private ExecutionResult executeProcessorStep(ProcessorExecutionStep executionSte if (onFailureExecutionSteps.isPresent()) { return executeSteps(onFailureExecutionSteps.get(), pipeline, doc, pipelineStopwatch); } else { - pipelineExecutionMetricsTracker.processorFailed(pipelineId, processorName, doc, processResult.getError()); + pipelineExecutionMetricsTracker.processorFailed(pipelineId, processorName, doc); return processorErrorExecutionResult(processResult.getError().get(), processorName, pipeline); } } diff --git a/sawmill-core/src/main/java/io/logz/sawmill/processors/AddSignatureProcessor.java b/sawmill-core/src/main/java/io/logz/sawmill/processors/AddSignatureProcessor.java index 3fdd431b..d53814bf 100644 --- a/sawmill-core/src/main/java/io/logz/sawmill/processors/AddSignatureProcessor.java +++ b/sawmill-core/src/main/java/io/logz/sawmill/processors/AddSignatureProcessor.java @@ -8,7 +8,10 @@ import io.logz.sawmill.exceptions.ProcessorExecutionException; import io.logz.sawmill.utilities.JsonUtils; import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -17,6 +20,7 @@ @ProcessorProvider(type = "addSignature", factory = AddSignatureProcessor.Factory.class) public class AddSignatureProcessor implements Processor { + private final static Logger logger = LoggerFactory.getLogger(AddSignatureProcessor.class); private final SignatureMode signatureMode; private final String signatureFieldName; private final Set includeValueFields; @@ -32,12 +36,14 @@ public ProcessResult process(Doc doc) throws InterruptedException { try { signature = createSignature(doc); } catch (Exception e) { - return ProcessResult.failure( - "failed to create signature, SignatureMode: " + signatureMode, + String errorMessage = "failed to create signature, SignatureMode: " + signatureMode; + logger.debug(errorMessage, e); + return ProcessResult.failure(errorMessage, new ProcessorExecutionException(AddSignatureProcessor.class.getSimpleName(), e)); } if(signature == 0) { + logger.debug("signature collection is empty, SignatureMode: " + signatureMode); if(signatureMode.equals(SignatureMode.FIELDS_NAMES) || signatureMode.equals(SignatureMode.HYBRID)) { return ProcessResult.failure("failed to extract fields names, SignatureMode: " + signatureMode); } @@ -72,7 +78,7 @@ private Set getFieldsValues(Doc doc) { return includeValueFields.stream() .filter(doc::hasField) .map(fieldName -> { - String value = doc.getField(fieldName); + String value = doc.getField(fieldName).toString(); return "value_" + fieldName + "_" + value; }).collect(Collectors.toSet()); } diff --git a/sawmill-core/src/main/java/io/logz/sawmill/processors/Base64DecodeProcessor.java b/sawmill-core/src/main/java/io/logz/sawmill/processors/Base64DecodeProcessor.java new file mode 100644 index 00000000..2ad8fdba --- /dev/null +++ b/sawmill-core/src/main/java/io/logz/sawmill/processors/Base64DecodeProcessor.java @@ -0,0 +1,68 @@ +package io.logz.sawmill.processors; + +import io.logz.sawmill.Doc; +import io.logz.sawmill.ProcessResult; +import io.logz.sawmill.Processor; +import io.logz.sawmill.annotations.ProcessorProvider; +import io.logz.sawmill.exceptions.ProcessorConfigurationException; +import io.logz.sawmill.utilities.JsonUtils; + +import java.util.Base64; +import java.util.Map; + +import static java.util.Objects.requireNonNull; + +@ProcessorProvider(type = "base64Decode", factory = Base64DecodeProcessor.Factory.class) +public class Base64DecodeProcessor implements Processor { + + private final String sourceField; + private final String targetField; + + public Base64DecodeProcessor(String sourceField, String targetField) { + this.sourceField = requireNonNull(sourceField); + this.targetField = requireNonNull(targetField); + } + + @Override + public ProcessResult process(Doc doc) { + if(!doc.hasField(sourceField, String.class)) + return ProcessResult.failure("field is missing from doc"); + + String value = doc.getField(sourceField); + String decodedValue = new String(Base64.getDecoder().decode(value)); + doc.addField(targetField, decodedValue); + return ProcessResult.success(); + } + + public static class Factory implements Processor.Factory { + public Factory() {} + + @Override + public Base64DecodeProcessor create(Map config) { + Base64DecodeProcessor.Configuration configuration = + JsonUtils.fromJsonMap(Base64DecodeProcessor.Configuration.class, config); + validateConfiguration(configuration); + return new Base64DecodeProcessor(configuration.getSourceField(), configuration.getTargetField()); + } + + private void validateConfiguration(Configuration configuration) { + if(configuration.getSourceField() == null || configuration.getSourceField().isEmpty() + || configuration.getTargetField() == null || configuration.getTargetField().isEmpty()) + throw new ProcessorConfigurationException("sourceField, targetField can not be null or empty"); + } + } + + public static class Configuration implements Processor.Configuration { + private String sourceField; + private String targetField; + + public Configuration() {} + public Configuration(String sourceField, String targetField) { + this.sourceField = sourceField; + this.targetField = targetField; + } + + public String getSourceField() { return sourceField; } + public String getTargetField() { return targetField; } + } +} diff --git a/sawmill-core/src/test/java/io/logz/sawmill/processors/AddSignatureProcessorTest.java b/sawmill-core/src/test/java/io/logz/sawmill/processors/AddSignatureProcessorTest.java index a8f96f83..367ecad9 100644 --- a/sawmill-core/src/test/java/io/logz/sawmill/processors/AddSignatureProcessorTest.java +++ b/sawmill-core/src/test/java/io/logz/sawmill/processors/AddSignatureProcessorTest.java @@ -349,7 +349,7 @@ private Set getFieldsValues(Doc doc, Set includeValueFields) { return includeValueFields.stream() .filter(doc::hasField) .map(fieldName -> { - String value = doc.getField(fieldName); + String value = doc.getField(fieldName).toString(); return "value_" + fieldName + "_" + value; }).collect(Collectors.toSet()); } diff --git a/sawmill-core/src/test/java/io/logz/sawmill/processors/Base64DecodeProcessorTest.java b/sawmill-core/src/test/java/io/logz/sawmill/processors/Base64DecodeProcessorTest.java new file mode 100644 index 00000000..2fcbda49 --- /dev/null +++ b/sawmill-core/src/test/java/io/logz/sawmill/processors/Base64DecodeProcessorTest.java @@ -0,0 +1,111 @@ +package io.logz.sawmill.processors; + +import io.logz.sawmill.Doc; +import io.logz.sawmill.ProcessResult; +import io.logz.sawmill.Processor; +import io.logz.sawmill.exceptions.ProcessorConfigurationException; +import org.junit.Test; + +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +import static io.logz.sawmill.utils.FactoryUtils.createProcessor; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class Base64DecodeProcessorTest { + + @Test + public void testNullOrEmptyConfigurationFieldsShouldFailCreator() { + Stream.of( + createConfig(null, "target"), + createConfig("", "target"), + createConfig("source", null), + createConfig("source", "")) + .forEach((config) -> assertThatThrownBy(() -> createProcessor(Base64DecodeProcessor.class, config)) + .isInstanceOf(ProcessorConfigurationException.class) + .hasMessageContaining("sourceField, targetField can not be null or empty")); + } + + @Test + public void testDecode() { + Map config = new HashMap<>(); + config.put("sourceField", "message"); + config.put("targetField", "message_decoded"); + + Map map = new HashMap<>(); + String encodedMessage = Base64.getEncoder().encodeToString("testEmptyFieldsShouldFail".getBytes()); + map.put("message", encodedMessage); + Doc doc = new Doc(map); + + ProcessResult result; + Processor processor = createProcessor(Base64DecodeProcessor.class, config); + try { + result = processor.process(doc); + } catch (InterruptedException e) { throw new RuntimeException(e); } + + assertThat(result != null && result.isSucceeded()).isTrue(); + assertThat(doc.getField(config.get("sourceField").toString()).toString()).isEqualTo(encodedMessage); + assertThat(doc.getField(config.get("targetField").toString()).toString()) + .isEqualTo("testEmptyFieldsShouldFail"); + } + + @Test + public void testNonStringFieldShouldFail() { + Map config = new HashMap<>(); + config.put("sourceField", "numberField"); + config.put("targetField", "noop"); + + Map map = new HashMap<>(); + map.put("message", "testSingleNonStringFieldShouldFail"); + map.put("numberField", 123); + Doc doc = new Doc(map); + + Processor processor = createProcessor(Base64DecodeProcessor.class, config); + + ProcessResult result; + + try { + result = processor.process(doc); + } catch (InterruptedException e) { throw new RuntimeException(e); } + assertThat(result != null && !result.isSucceeded()).isTrue(); + assertThat(result.getError().isPresent()).isTrue(); + assertThat(result.getError().get().getMessage()) + .isEqualTo("field is missing from doc"); + assertThat(doc.hasField(config.get("targetField").toString())).isFalse(); + assertThat(doc.getField("message").toString()).isEqualTo(map.get("message").toString()); + } + + @Test + public void testMissingFieldShouldFail() { + Map config = new HashMap<>(); + config.put("sourceField", "foo"); + config.put("targetField", "foo_decoded"); + + Map map = new HashMap<>(); + map.put("message", "testAllFieldsMissingShouldFail"); + + Doc doc = new Doc(map); + + ProcessResult result; + Processor processor = createProcessor(Base64DecodeProcessor.class, config); + try { + result = processor.process(doc); + } catch (InterruptedException e) { throw new RuntimeException(e); } + + assertThat(result != null && !result.isSucceeded()).isTrue(); + assertThat(result.getError().get().getMessage()) + .isEqualTo("field is missing from doc"); + assertThat(doc.hasField(config.get("targetField").toString())).isFalse(); + assertThat(doc.getField("message").toString()).isEqualTo(map.get("message").toString()); + } + + private Map createConfig(String sourceField, String targetField) { + Map config = new HashMap<>(); + config.put("sourceField", sourceField); + config.put("targetField", targetField); + return config; + } +}