Report native Weaver test automation exclusion declarations

dd-java-agent/instrumentation/weaver-0.9/build.gradle

@@ -5,6 +5,18 @@ plugins {
 }
 
 muzzle {
+  ['2.12', '2.13'].each { scalaVersion ->
+    pass {
+      group = 'com.disneystreaming'
+      module = "weaver-cats_${scalaVersion}"
+      versions = '[0.8.4,)'
+    }
+    pass {
+      group = 'org.typelevel'
+      module = "weaver-cats_${scalaVersion}"
+      versions = '[0.9.0,)'
+    }
+  }
   pass {
     group = 'com.disneystreaming'
     module = 'weaver-cats_3'
@@ -49,3 +61,29 @@ tasks.named("compileWeaver084TestGroovy", GroovyCompile) {
   dependsOn 'compileWeaver084TestScala'
   classpath += files(sourceSets.weaver084Test.scala.destinationDirectory)
 }
+
+// Scala 2 emits static suite-loaded callbacks with the task as its first argument.
+['scala212Test', 'scala213Test'].each { suiteName ->
+  addTestSuiteForDir(suiteName, 'scala2Test')
+  sourceSets.named(suiteName) {
+    scala {
+      srcDirs = ['src/test/scala']
+      include 'org/example/TestAutomationExcluded.scala', 'org/example/TestAutomationNamed.scala'
+    }
+  }
+  configurations.matching { it.name.startsWith(suiteName) }.configureEach {
+    exclude group: 'org.typelevel', module: 'weaver-cats_3'
+  }
+  tasks.named("compile${suiteName.capitalize()}Groovy", GroovyCompile) {
+    dependsOn "compile${suiteName.capitalize()}Scala"
+    classpath += files(sourceSets[suiteName].scala.destinationDirectory)
+  }
+  tasks.named("compile${suiteName.capitalize()}Scala", ScalaCompile) {
+    configureCompiler(it, 8, JavaVersion.VERSION_1_8, 'Scala 2 compiler compatibility')
+  }
+}
+
+dependencies {
+  scala212TestImplementation group: 'org.typelevel', name: 'weaver-cats_2.12', version: '0.9.0'
+  scala213TestImplementation group: 'org.typelevel', name: 'weaver-cats_2.13', version: '0.9.0'
+}

dd-java-agent/instrumentation/weaver-0.9/src/main/java/datadog/trace/instrumentation/weaver/DatadogWeaverReporter.java

@@ -10,11 +10,18 @@
 import datadog.trace.api.time.SystemTimeSource;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import java.lang.reflect.Method;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import sbt.testing.TaskDef;
 import scala.Option;
 import weaver.Result;
+import weaver.TestName;
 import weaver.TestOutcome;
 import weaver.framework.RunEvent;
 import weaver.framework.SuiteFinished;
@@ -24,6 +31,38 @@
 public class DatadogWeaverReporter {
 
   private static final String TEST_FRAMEWORK = "weaver";
+  private static final Logger LOGGER = LoggerFactory.getLogger(DatadogWeaverReporter.class);
+  private static final Map<String, Map<String, Collection<String>>> TEST_TAGS =
+      new ConcurrentHashMap<>();
+
+  /** Takes a fresh snapshot per execution and retains only names and strings. */
+  public static void registerTestTags(String suiteName, Object suite) {
+    try {
+      scala.collection.Iterable<?> plan =
+          (scala.collection.Iterable<?>) suite.getClass().getMethod("plan").invoke(suite);
+      Map<String, Collection<String>> tagsByName = new HashMap<>();
+      scala.collection.Iterator<?> tests = plan.iterator();
+      while (tests.hasNext()) {
+        TestName testName = (TestName) tests.next();
+        Collection<String> tags = new ArrayList<>();
+        scala.collection.Iterator<String> declaredTags = testName.tags().iterator();
+        while (declaredTags.hasNext()) {
+          String tag = declaredTags.next();
+          if ("datadog_no_auto_retry".equals(tag)
+              || "datadog_prevent_failure_suppression".equals(tag)) {
+            tags.add(tag);
+          }
+        }
+        if (!tags.isEmpty()) {
+          tagsByName.put(testName.name(), tags);
+        }
+      }
+      TEST_TAGS.put(suiteName, tagsByName);
+    } catch (Exception e) {
+      LOGGER.debug("Could not read Weaver test declaration tags", e);
+    }
+  }
+
   private static final String TEST_FRAMEWORK_VERSION = WeaverUtils.getWeaverVersion();
 
   private static volatile TestEventsHandler<TestSuiteDescriptor, TestDescriptor>
@@ -50,6 +89,7 @@ public static synchronized void stop() {
     if (TEST_EVENTS_HANDLER != null) {
       TEST_EVENTS_HANDLER.close();
       TEST_EVENTS_HANDLER = null;
+      TEST_TAGS.clear();
     }
   }
 
@@ -89,6 +129,7 @@ public static void onSuiteFinish(SuiteFinished event) {
     Class<?> testClass = WeaverUtils.getClass(testSuiteName);
 
     TEST_EVENTS_HANDLER.onTestSuiteFinish(new TestSuiteDescriptor(testSuiteName, testClass), null);
+    TEST_TAGS.remove(testSuiteName);
   }
 
   public static void onTestFinished(TestFinished event, TaskDef taskDef) {
@@ -104,7 +145,10 @@ public static void onTestFinished(TestFinished event, TaskDef taskDef) {
     String testName = event.outcome().name();
     Object testQualifier = null;
     String testParameters = null;
-    Collection<String> categories = Collections.emptyList();
+    Collection<String> categories =
+        TEST_TAGS
+            .getOrDefault(testSuiteName, Collections.emptyMap())
+            .getOrDefault(testName, Collections.emptyList());
     TestDescriptor testDescriptor =
         new TestDescriptor(testSuiteName, testClass, testName, testParameters, testQualifier);
     String testMethodName = null;

dd-java-agent/instrumentation/weaver-0.9/src/main/java/datadog/trace/instrumentation/weaver/WeaverTestTagsInstrumentation.java

@@ -0,0 +1,64 @@
+package datadog.trace.instrumentation.weaver;
+
+import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.isStatic;
+import static net.bytebuddy.matcher.ElementMatchers.none;
+import static net.bytebuddy.matcher.ElementMatchers.not;
+import static net.bytebuddy.matcher.ElementMatchers.takesArgument;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+
+import com.google.auto.service.AutoService;
+import datadog.trace.agent.tooling.Instrumenter;
+import datadog.trace.agent.tooling.InstrumenterModule;
+import net.bytebuddy.asm.Advice;
+import net.bytebuddy.description.method.MethodDescription;
+import net.bytebuddy.matcher.ElementMatcher;
+
+/** Captures declaration tags with the task identity used by Weaver's reporting events. */
+@AutoService(InstrumenterModule.class)
+public class WeaverTestTagsInstrumentation extends InstrumenterModule.CiVisibility
+    implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice {
+  public WeaverTestTagsInstrumentation() {
+    super("ci-visibility", "weaver");
+  }
+
+  @Override
+  public String instrumentedType() {
+    return "weaver.framework.RunnerCompat$IOTask";
+  }
+
+  @Override
+  public ElementMatcher<? super MethodDescription> methodIgnoreMatcher() {
+    return none();
+  }
+
+  @Override
+  public void methodAdvice(MethodTransformer transformer) {
+    // The suite-loaded flatMap callback receives the actual suite returned by its provider.
+    transformer.applyAdvice(
+        takesArguments(2).and(not(isStatic())).and(takesArgument(1, named("weaver.EffectSuite"))),
+        WeaverTestTagsInstrumentation.class.getName() + "$CaptureTagsAdvice");
+    // Scala 2 lifts the receiver into the first argument of its static callback.
+    transformer.applyAdvice(
+        isStatic()
+            .and(takesArguments(3))
+            .and(takesArgument(0, named(instrumentedType())))
+            .and(takesArgument(2, named("weaver.EffectSuite"))),
+        WeaverTestTagsInstrumentation.class.getName() + "$CaptureScala2TagsAdvice");
+  }
+
+  public static class CaptureTagsAdvice {
+    @Advice.OnMethodEnter(suppress = Throwable.class)
+    public static void capture(
+        @Advice.FieldValue("fqn") String suiteName, @Advice.Argument(1) Object suite) {
+      DatadogWeaverReporter.registerTestTags(suiteName, suite);
+    }
+  }
+
+  public static class CaptureScala2TagsAdvice {
+    @Advice.OnMethodEnter(suppress = Throwable.class)
+    public static void capture(@Advice.Argument(0) Object task, @Advice.Argument(2) Object suite) {
+      DatadogWeaverReporter.registerTestTags(WeaverUtils.getTaskName(task), suite);
+    }
+  }
+}

dd-java-agent/instrumentation/weaver-0.9/src/main/java/datadog/trace/instrumentation/weaver/WeaverUtils.java

@@ -41,6 +41,14 @@ public abstract class WeaverUtils {
   private static final MethodHandle GET_EXPECTATION_FAILED_MESSAGE_HANDLE =
       METHOD_HANDLES.method(getClass(EXPECTATION_FAILED_CLASS_NAME), "message");
 
+  // IOTask is private in the Scala 2 build of Weaver 0.8.4.
+  private static final MethodHandle GET_TASK_NAME_HANDLE =
+      METHOD_HANDLES.method(getClass("weaver.framework.RunnerCompat$IOTask"), "fqn");
+
+  public static String getTaskName(Object task) {
+    return METHOD_HANDLES.invoke(GET_TASK_NAME_HANDLE, task);
+  }
+
   public static final List<LibraryCapability> CAPABILITIES = Collections.emptyList();
 
   private WeaverUtils() {}

dd-java-agent/instrumentation/weaver-0.9/src/scala2Test/groovy/WeaverScala2Test.groovy

@@ -0,0 +1,87 @@
+import datadog.trace.api.DisableTestTrace
+import datadog.trace.civisibility.CiVisibilityInstrumentationTest
+import datadog.trace.instrumentation.weaver.DatadogWeaverReporter
+import datadog.trace.instrumentation.weaver.WeaverUtils
+import org.example.TestAutomationExcluded
+import org.example.TestAutomationNamed
+import org.example.TestAutomationProvider
+import org.slf4j.LoggerFactory
+import sbt.testing.EventHandler
+import sbt.testing.Logger
+import sbt.testing.SubclassFingerprint
+import sbt.testing.SuiteSelector
+import sbt.testing.TaskDef
+import weaver.framework.CatsEffect
+
+@DisableTestTrace(reason = 'avoid self-tracing')
+class WeaverScala2Test extends CiVisibilityInstrumentationTest {
+  def "Scala 2 declarations preserve exclusion tags and native failure and skip outcomes"() {
+    when:
+    runNativeSuites([TestAutomationExcluded.name])
+
+    then:
+    def spans = TEST_WRITER.toList().flatten().findAll { it.getTag('test.name') in ['protected_failure', 'protected_skip'] }
+    spans.size() == 2
+    spans.every { it.getTag('test.test_management.no_auto_retry') == true }
+    spans.every { it.getTag('test.test_management.prevent_failure_suppression') == true }
+    spans.find { it.getTag('test.name') == 'protected_failure' }.getTag('test.status').toString() == 'fail'
+    spans.find { it.getTag('test.name') == 'protected_skip' }.getTag('test.status').toString() == 'skip'
+  }
+
+  def "Scala 2 task identities preserve distinct provider declarations"() {
+    when:
+    runNativeSuites([TestAutomationNamed.name, TestAutomationProvider.name])
+
+    then:
+    def spans = TEST_WRITER.toList().flatten().findAll { it.getTag('test.name') == 'named_test' }
+    spans.size() == 2
+    def named = spans.find { it.getTag('test.suite') == TestAutomationNamed.name }
+    named.getTag('test.test_management.no_auto_retry') == true
+    named.getTag('test.test_management.prevent_failure_suppression') == null
+    def provided = spans.find { it.getTag('test.suite') == TestAutomationProvider.name }
+    provided.getTag('test.test_management.prevent_failure_suppression') == true
+    provided.getTag('test.test_management.no_auto_retry') == null
+  }
+
+  private void runNativeSuites(List<String> names) {
+    assert CatsEffect.protectionDomain.codeSource.location.toString().contains('weaver-cats_2.')
+    def output = new ByteArrayOutputStream()
+    def stream = new PrintStream(output, false, 'UTF-8')
+    def framework = new CatsEffect(stream)
+    def runner = framework.runner([] as String[], [] as String[], getClass().classLoader)
+    def fingerprint = framework.fingerprints().find { it instanceof SubclassFingerprint && it.isModule() && it.superclassName() == 'weaver.BaseCatsSuite' }
+    def definitions = names.collect { new TaskDef(it, fingerprint, false, [new SuiteSelector()] as sbt.testing.Selector[]) } as TaskDef[]
+    def events = []
+    def handler = { event -> events.add(event) } as EventHandler
+    def log = LoggerFactory.getLogger(WeaverScala2Test)
+    def logger = [
+      ansiCodesSupported: { false },
+      error: { String message -> log.error(message) },
+      warn: { String message -> log.warn(message) },
+      info: { String message -> log.info(message) },
+      debug: { String message -> log.debug(message) },
+      trace: { Throwable error ->
+        log.error('Weaver error', error)
+      }
+    ] as Logger
+    DatadogWeaverReporter.start()
+    try {
+      runner.tasks(definitions).each { it.execute(handler, [logger] as Logger[]) }
+      assert !events.empty
+    } finally {
+      runner.done()
+      stream.close()
+      DatadogWeaverReporter.stop()
+    }
+  }
+
+  @Override
+  String instrumentedLibraryName() {
+    'weaver'
+  }
+
+  @Override
+  String instrumentedLibraryVersion() {
+    WeaverUtils.weaverVersion
+  }
+}

dd-java-agent/instrumentation/weaver-0.9/src/test/groovy/WeaverTest.groovy

@@ -5,6 +5,9 @@ import datadog.trace.instrumentation.weaver.WeaverIntegrationTestRunner
 import datadog.trace.instrumentation.weaver.WeaverUtils
 import org.example.TestCanceled
 import org.example.TestFailed
+import org.example.TestAutomationExcluded
+import org.example.TestAutomationNamed
+import org.example.TestAutomationProvider
 import org.example.TestFailedExceptionPure
 import org.example.TestFailedPure
 import org.example.TestIgnored
@@ -16,6 +19,48 @@ import org.example.TestSucceedSuiteResource
 @DisableTestTrace(reason = "avoid self-tracing")
 class WeaverTest extends CiVisibilityInstrumentationTest {
 
+  def "test automation exclusions retain native outcomes and declaration tags"() {
+    when:
+    runTests([TestAutomationExcluded])
+
+    then:
+    def testSpans = TEST_WRITER.toList().flatten().findAll { it.getTag("test.name") == "protected_failure" || it.getTag("test.name") == "protected_skip" }
+    testSpans.size() == 2
+    def ordinary = TEST_WRITER.toList().flatten().find { it.getTag("test.name") == "ordinary_tagged" }
+    ordinary != null
+    ordinary.getTag("test.test_management.no_auto_retry") == null
+    ordinary.getTag("test.test_management.prevent_failure_suppression") == null
+    ordinary.getTag("test.traits") == null
+    testSpans.every { it.getTag("test.test_management.no_auto_retry") == true }
+    testSpans.every { it.getTag("test.test_management.prevent_failure_suppression") == true }
+    testSpans.find { it.getTag("test.name") == "protected_failure" }.getTag("test.status").toString() == "fail"
+    testSpans.find { it.getTag("test.name") == "protected_skip" }.getTag("test.status").toString() == "skip"
+  }
+
+  def "custom suite and provider identities preserve distinct declaration tags"() {
+    when:
+    DatadogWeaverReporter.start()
+    WeaverIntegrationTestRunner.runTests([TestAutomationNamed.name, TestAutomationProvider.name])
+
+    then:
+    def spans = TEST_WRITER.toList().flatten().findAll { it.getTag("test.name") == "named_test" }
+    spans.size() == 2
+    def named = spans.find { it.getTag("test.suite") == TestAutomationNamed.name }
+    named.getTag("test.test_management.no_auto_retry") == true
+    named.getTag("test.test_management.prevent_failure_suppression") == null
+    def provided = spans.find { it.getTag("test.suite") == TestAutomationProvider.name }
+    provided.getTag("test.test_management.prevent_failure_suppression") == true
+    provided.getTag("test.test_management.no_auto_retry") == null
+
+    and:
+    def snapshots = DatadogWeaverReporter.getDeclaredField("TEST_TAGS")
+    snapshots.setAccessible(true)
+    snapshots.get(null).isEmpty()
+
+    cleanup:
+    DatadogWeaverReporter.stop()
+  }
+
   def "test #testcaseName"() {
     runTests(tests)
 

dd-java-agent/instrumentation/weaver-0.9/src/test/scala/org/example/TestAutomationExcluded.scala

@@ -0,0 +1,23 @@
+package org.example
+
+import cats.effect.IO
+import datadog.trace.api.civisibility.TestAutomationExclusions
+import weaver._
+
+object TestAutomationExcluded extends SimpleIOSuite {
+  private def excluded(name: String): TestName = TestName
+    .fromString(name)
+    .tagged(TestAutomationExclusions.NO_AUTO_RETRY)
+    .tagged(TestAutomationExclusions.PREVENT_FAILURE_SUPPRESSION)
+
+  test(excluded("protected_failure")) {
+    IO.pure(expect(false))
+  }
+
+  test(excluded("protected_skip")) {
+    ignore("developer skip")
+  }
+  test("ordinary_tagged".tagged("customer_tag")) {
+    IO.pure(expect(true))
+  }
+}