From 8e8a67ed4abfe780a50b614813e0d1d9299a289a Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Thu, 26 Sep 2024 19:56:09 +0000 Subject: [PATCH 001/162] Update scala-library, scala-reflect to 2.13.15 --- .github/workflows/ci.yml | 2 +- project/Dependencies.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 272ffaecd..3f7ebe932 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - scala: [ 2.13.14 ] + scala: [ 2.13.15 ] command: [ udash-jvm/test, udash-js/test, guide-selenium/test ] steps: - uses: actions/checkout@v4 diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 163981056..96c64123e 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -5,7 +5,7 @@ import sbt.* import sbt.Keys.scalaVersion object Dependencies { - val versionOfScala = "2.13.14" //update .github/workflows/ci.yml as well + val versionOfScala = "2.13.15" //update .github/workflows/ci.yml as well val jqueryWrapperVersion = "3.3.0" From ea448f3dc2d054249a57bc307e70eed6e7d07867 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 2 Oct 2024 20:09:16 +0000 Subject: [PATCH 002/162] Update commons-analyzer, commons-core, ... to 2.20.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index a193a105e..7e52cd9f4 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -14,7 +14,7 @@ object Dependencies { val scalaCssVersion = "1.0.0" val servletVersion = "4.0.1" - val avsCommonsVersion = "2.19.0" + val avsCommonsVersion = "2.20.0" val atmosphereJSVersion = "3.1.3" val atmosphereVersion = "2.7.14" From 2b7e8a7c7158c362e53d85b91d475c00d7952c61 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 2 Oct 2024 20:09:21 +0000 Subject: [PATCH 003/162] Update jetty-client, jetty-rewrite, ... to 12.0.14 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index a193a105e..702f1d1fa 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -28,7 +28,7 @@ object Dependencies { val scalaLoggingVersion = "3.9.5" - val jettyVersion = "12.0.13" + val jettyVersion = "12.0.14" val typesafeConfigVersion = "1.4.3" val flexmarkVersion = "0.64.8" val logbackVersion = "1.3.14" From d417426c4e53bb6c23d94b3f275c0a9c04635b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Kozak?= Date: Thu, 3 Oct 2024 11:02:56 +0200 Subject: [PATCH 004/162] update ScalaJs to 1.17.0, fix TestRPC (inferred Any) --- project/plugins.sbt | 2 +- rpc/src/test/scala/io/udash/rpc/TestRPC.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 2e0af8190..4a8654d2f 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -3,7 +3,7 @@ logLevel := Level.Warn libraryDependencies += "org.scala-js" %% "scalajs-env-selenium" % "1.1.1" libraryDependencies += "org.scala-js" %% "scalajs-env-jsdom-nodejs" % "1.1.0" -addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.16.0") +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.17.0") addSbtPlugin("org.scala-js" % "sbt-jsdependencies" % "1.0.2") addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.4") addSbtPlugin("org.jetbrains.scala" % "sbt-ide-settings" % "1.1.2") diff --git a/rpc/src/test/scala/io/udash/rpc/TestRPC.scala b/rpc/src/test/scala/io/udash/rpc/TestRPC.scala index 55dfdebc8..1c0c1e0f2 100644 --- a/rpc/src/test/scala/io/udash/rpc/TestRPC.scala +++ b/rpc/src/test/scala/io/udash/rpc/TestRPC.scala @@ -108,7 +108,7 @@ trait RPCMethodsImpl extends RPCMethods { onFire("handleMore", List(Nil)) override def doStuff(lol: Int, fuu: String)(cos: Option[Boolean]): Unit = - onFire("doStuff", List(List(lol, fuu), List(cos))) + onFire("doStuff", List(List[Any](lol, fuu), List(cos))) override def doStuff(num: Int): Unit = onFire("doStuffInt", List(List(num))) From 05922a3f3b7c305087b789377a3fa91c7cf1b35c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Kozak?= Date: Thu, 3 Oct 2024 17:34:04 +0200 Subject: [PATCH 005/162] sttp version bump 3.10.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6f348a985..c5718b4e7 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -24,7 +24,7 @@ object Dependencies { val circeDerivationVersion = "0.13.0-M5" // Tests only val monixVersion = "3.4.1" // udash-rest only - val sttpVersion = "3.9.8" + val sttpVersion = "3.10.0" val scalaLoggingVersion = "3.9.5" From 2ad1022d333a19f442acf8417b856fc3632bce5b Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sun, 6 Oct 2024 17:45:39 +0000 Subject: [PATCH 006/162] Update sbt-pgp to 2.3.0 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 2e0af8190..411f584fd 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -11,5 +11,5 @@ addSbtPlugin("com.github.sbt" % "sbt-less" % "2.0.1") addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.10.4") // Deployment configuration -addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.2.1") +addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.0") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.11.3") \ No newline at end of file From 68cb9fda9f78191594cb567ab1821d18a7096d07 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 8 Oct 2024 22:41:25 +0000 Subject: [PATCH 007/162] Update atmosphere-runtime to 2.7.15 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 7ef9ab952..1028a87f9 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -17,7 +17,7 @@ object Dependencies { val avsCommonsVersion = "2.20.0" val atmosphereJSVersion = "3.1.3" - val atmosphereVersion = "2.7.14" + val atmosphereVersion = "2.7.15" val upickleVersion = "4.0.2" // Tests only val circeVersion = "0.14.10" // Tests only From 20dc58625ea5730a746883c839d8d4daa39d07c7 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 12 Oct 2024 19:27:27 +0000 Subject: [PATCH 008/162] Update sbt-sonatype to 3.12.2 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index ac75f34e1..cd30ca368 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -12,4 +12,4 @@ addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.10.4") // Deployment configuration addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.0") -addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.11.3") \ No newline at end of file +addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.12.2") \ No newline at end of file From d8a5e9b3e658f7af5eb45f516bbdf01d988a3603 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 16 Oct 2024 23:52:43 +0000 Subject: [PATCH 009/162] Update monix to 3.10.1 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 1028a87f9..f5121712e 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -24,7 +24,7 @@ object Dependencies { val circeDerivationVersion = "0.13.0-M5" // Tests only val monixVersion = "3.4.1" // udash-rest only - val sttpVersion = "3.10.0" + val sttpVersion = "3.10.1" val scalaLoggingVersion = "3.9.5" From c87963eef38829fec18bb23dd29120e972d9cb51 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sun, 20 Oct 2024 23:50:29 +0000 Subject: [PATCH 010/162] Update sbt, scripted-plugin to 1.10.3 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index 526f82fa7..51bbc4f00 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ # suppress inspection "UnusedProperty" -sbt.version=1.10.2 +sbt.version=1.10.3 From 14f81ac25dae589b5f84b4b41dcd019a2e5d8bb6 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Mon, 28 Oct 2024 20:38:25 +0000 Subject: [PATCH 011/162] Update sbt, scripted-plugin to 1.10.4 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index 51bbc4f00..d41ad991a 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ # suppress inspection "UnusedProperty" -sbt.version=1.10.3 +sbt.version=1.10.4 From a459c5c9de7b549d0988de0e325d16de7b50f5a8 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Thu, 31 Oct 2024 02:33:20 +0000 Subject: [PATCH 012/162] Update selenium-java to 4.26.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index f5121712e..fc6a7ce84 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -42,7 +42,7 @@ object Dependencies { val bootstrap4DatepickerVersion = "5.39.0" val momentJsVersion = "2.30.1" - val seleniumVersion = "4.25.0" + val seleniumVersion = "4.26.0" val webDriverManagerVersion = "5.9.2" val scalaJsBenchmarkVersion = "0.10.0" From c5cf6243c07e3bfc0c90d9c63e891aeda489a95f Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 6 Nov 2024 03:53:48 +0000 Subject: [PATCH 013/162] Update sbt, scripted-plugin to 1.10.5 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index d41ad991a..aba0a87c5 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ # suppress inspection "UnusedProperty" -sbt.version=1.10.4 +sbt.version=1.10.5 From 182b9d68668cde37ec5c8a4ee2363f0ecdcbc7cc Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Thu, 7 Nov 2024 20:10:46 +0000 Subject: [PATCH 014/162] Update jetty-ee8-servlet to 12.0.15 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index fc6a7ce84..16c1e8e76 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -28,7 +28,7 @@ object Dependencies { val scalaLoggingVersion = "3.9.5" - val jettyVersion = "12.0.14" + val jettyVersion = "12.0.15" val typesafeConfigVersion = "1.4.3" val flexmarkVersion = "0.64.8" val logbackVersion = "1.3.14" From a5fcf449cf41dbcdbe78c802b217079030a1476f Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Mon, 25 Nov 2024 19:43:43 +0000 Subject: [PATCH 015/162] Update selenium-java to 4.27.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 16c1e8e76..8d6e3407b 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -42,7 +42,7 @@ object Dependencies { val bootstrap4DatepickerVersion = "5.39.0" val momentJsVersion = "2.30.1" - val seleniumVersion = "4.26.0" + val seleniumVersion = "4.27.0" val webDriverManagerVersion = "5.9.2" val scalaJsBenchmarkVersion = "0.10.0" From d75f8b8af9e76d84b15d226e0721d3f24a06ba50 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Mon, 2 Dec 2024 03:39:57 +0000 Subject: [PATCH 016/162] Update sbt, scripted-plugin to 1.10.6 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index aba0a87c5..bf2ef99c4 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ # suppress inspection "UnusedProperty" -sbt.version=1.10.5 +sbt.version=1.10.6 From 9858e567b2bcd414740678f31912442c04c998cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Kozak?= Date: Wed, 11 Dec 2024 13:58:31 +0100 Subject: [PATCH 017/162] close jetty connections on monix timeout (or other cancellation) --- .../io/udash/rest/jetty/JettyRestClient.scala | 105 +++++++++--------- ...eStaleJettyConnectionsOnMonixTimeout.scala | 82 ++++++++++++++ .../scala/io/udash/rest/RestTestApi.scala | 10 +- 3 files changed, 141 insertions(+), 56 deletions(-) create mode 100644 rest/jetty/src/test/scala/io/udash/rest/jetty/CloseStaleJettyConnectionsOnMonixTimeout.scala diff --git a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala index ed8c4e6af..86f1118a1 100644 --- a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala +++ b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala @@ -1,17 +1,18 @@ package io.udash package rest.jetty -import com.avsystem.commons._ +import com.avsystem.commons.* import com.avsystem.commons.annotation.explicitGenerics -import io.udash.rest.raw._ +import io.udash.rest.raw.* import io.udash.utils.URLEncoder import monix.eval.Task -import org.eclipse.jetty.client.{BufferingResponseListener, BytesRequestContent, HttpClient, Result, StringRequestContent} +import monix.execution.Callback +import org.eclipse.jetty.client.* import org.eclipse.jetty.http.{HttpCookie, HttpHeader, MimeTypes} import java.nio.charset.Charset -import scala.concurrent.duration._ -import scala.util.{Failure, Success} +import scala.concurrent.CancellationException +import scala.concurrent.duration.* object JettyRestClient { final val DefaultMaxResponseLength = 2 * 1024 * 1024 @@ -31,55 +32,57 @@ object JettyRestClient { maxResponseLength: Int = DefaultMaxResponseLength, timeout: Duration = DefaultTimeout ): RawRest.HandleRequest = - request => Task.async { callback => - val path = baseUrl + PlainValue.encodePath(request.parameters.path) - val httpReq = client.newRequest(baseUrl).method(request.method.name) + request => Task(client.newRequest(baseUrl).method(request.method.name)).flatMap { httpReq => + Task.async { (callback: Callback[Throwable, RestResponse]) => + val path = baseUrl + PlainValue.encodePath(request.parameters.path) - httpReq.path(path) - request.parameters.query.entries.foreach { - case (name, PlainValue(value)) => httpReq.param(name, value) - } - request.parameters.headers.entries.foreach { - case (name, PlainValue(value)) => httpReq.headers(headers => headers.add(name, value)) - } - request.parameters.cookies.entries.foreach { - case (name, PlainValue(value)) => httpReq.cookie(HttpCookie.build( - URLEncoder.encode(name, spaceAsPlus = true), URLEncoder.encode(value, spaceAsPlus = true)).build()) - } - - request.body match { - case HttpBody.Empty => - case tb: HttpBody.Textual => - httpReq.body(new StringRequestContent(tb.contentType, tb.content, Charset.forName(tb.charset))) - case bb: HttpBody.Binary => - httpReq.body(new BytesRequestContent(bb.contentType, bb.bytes)) - } + httpReq.path(path) + request.parameters.query.entries.foreach { + case (name, PlainValue(value)) => httpReq.param(name, value) + } + request.parameters.headers.entries.foreach { + case (name, PlainValue(value)) => httpReq.headers(headers => headers.add(name, value)) + } + request.parameters.cookies.entries.foreach { + case (name, PlainValue(value)) => httpReq.cookie(HttpCookie.build( + URLEncoder.encode(name, spaceAsPlus = true), URLEncoder.encode(value, spaceAsPlus = true)).build()) + } - timeout match { - case fd: FiniteDuration => httpReq.timeout(fd.length, fd.unit) - case _ => - } + request.body match { + case HttpBody.Empty => + case tb: HttpBody.Textual => + httpReq.body(new StringRequestContent(tb.contentType, tb.content, Charset.forName(tb.charset))) + case bb: HttpBody.Binary => + httpReq.body(new BytesRequestContent(bb.contentType, bb.bytes)) + } - httpReq.send(new BufferingResponseListener(maxResponseLength) { - override def onComplete(result: Result): Unit = - if (result.isSucceeded) { - val httpResp = result.getResponse - val contentTypeOpt = httpResp.getHeaders.get(HttpHeader.CONTENT_TYPE).opt - val charsetOpt = contentTypeOpt.map(MimeTypes.getCharsetFromContentType) - val body = (contentTypeOpt, charsetOpt) match { - case (Opt(contentType), Opt(charset)) => - HttpBody.textual(getContentAsString, MimeTypes.getContentTypeWithoutCharset(contentType), charset) - case (Opt(contentType), Opt.Empty) => - HttpBody.binary(getContent, contentType) - case _ => - HttpBody.Empty - } - val headers = httpResp.getHeaders.asScala.iterator.map(h => (h.getName, PlainValue(h.getValue))).toList - val response = RestResponse(httpResp.getStatus, IMapping(headers), body) - callback(Success(response)) - } else { - callback(Failure(result.getFailure)) + timeout match { + case fd: FiniteDuration => httpReq.timeout(fd.length, fd.unit) + case _ => } - }) + + httpReq.send(new BufferingResponseListener(maxResponseLength) { + override def onComplete(result: Result): Unit = + if (result.isSucceeded) { + val httpResp = result.getResponse + val contentTypeOpt = httpResp.getHeaders.get(HttpHeader.CONTENT_TYPE).opt + val charsetOpt = contentTypeOpt.map(MimeTypes.getCharsetFromContentType) + val body = (contentTypeOpt, charsetOpt) match { + case (Opt(contentType), Opt(charset)) => + HttpBody.textual(getContentAsString, MimeTypes.getContentTypeWithoutCharset(contentType), charset) + case (Opt(contentType), Opt.Empty) => + HttpBody.binary(getContent, contentType) + case _ => + HttpBody.Empty + } + val headers = httpResp.getHeaders.asScala.iterator.map(h => (h.getName, PlainValue(h.getValue))).toList + val response = RestResponse(httpResp.getStatus, IMapping(headers), body) + callback(Success(response)) + } else { + callback(Failure(result.getFailure)) + } + }) + } + .doOnCancel(Task(httpReq.abort(new CancellationException("Request cancelled")))) } } diff --git a/rest/jetty/src/test/scala/io/udash/rest/jetty/CloseStaleJettyConnectionsOnMonixTimeout.scala b/rest/jetty/src/test/scala/io/udash/rest/jetty/CloseStaleJettyConnectionsOnMonixTimeout.scala new file mode 100644 index 000000000..af8c764b2 --- /dev/null +++ b/rest/jetty/src/test/scala/io/udash/rest/jetty/CloseStaleJettyConnectionsOnMonixTimeout.scala @@ -0,0 +1,82 @@ +package io.udash.rest.jetty + +import com.avsystem.commons.misc.ScalaDurationExtensions.durationIntOps +import com.avsystem.commons.universalOps +import io.udash.rest.jetty.CloseStaleJettyConnectionsOnMonixTimeout.RestApiWithNeverCounter +import io.udash.rest.{DefaultRestApiCompanion, GET, RestServlet} +import monix.eval.Task +import monix.execution.atomic.Atomic +import org.eclipse.jetty.client.HttpClient +import org.eclipse.jetty.ee8.servlet.{ServletContextHandler, ServletHolder} +import org.eclipse.jetty.server.{NetworkConnector, Server} +import org.scalatest.funsuite.AsyncFunSuite + +import java.net.InetSocketAddress +import scala.concurrent.Future +import scala.concurrent.duration.{FiniteDuration, IntMult} + +final class CloseStaleJettyConnectionsOnMonixTimeout extends AsyncFunSuite { + + test("close connection on monix task timeout") { + import monix.execution.Scheduler.Implicits.global + + val MaxConnections: Int = 1 // to timeout quickly + val Connections: Int = 10 // > MaxConnections + val RequestTimeout: FiniteDuration = 1.hour // no timeout + val CallTimeout: FiniteDuration = 300.millis + + + val server = new Server(new InetSocketAddress("localhost", 0)) { + setHandler( + new ServletContextHandler().setup( + _.addServlet( + new ServletHolder( + RestServlet[RestApiWithNeverCounter](RestApiWithNeverCounter.Impl) + ), + "/*", + ) + ) + ) + start() + } + + val httpClient = new HttpClient() { + setMaxConnectionsPerDestination(MaxConnections) + setIdleTimeout(RequestTimeout.toMillis) + start() + } + + val client = JettyRestClient[RestApiWithNeverCounter]( + client = httpClient, + baseUri = server.getConnectors.head |> { case connector: NetworkConnector => s"http://${connector.getHost}:${connector.getLocalPort}" }, + maxResponseLength = Int.MaxValue, // to avoid unnecessary logs + timeout = RequestTimeout, + ) + + Task + .traverse(List.range(0, Connections))(_ => Task.fromFuture(client.neverGet).timeout(CallTimeout).failed) + .timeoutTo(Connections * CallTimeout + 500.millis, Task(fail("All connections should have been closed"))) // + 500 millis just in case + .map(_ => assert(RestApiWithNeverCounter.Impl.counter.get() == Connections)) // neverGet should be called Connections times + .guarantee(Task { + server.stop() + httpClient.stop() + }) + .runToFuture + } +} + +object CloseStaleJettyConnectionsOnMonixTimeout { + sealed trait RestApiWithNeverCounter { + final val counter = Atomic(0) + @GET def neverGet: Future[Unit] + } + + object RestApiWithNeverCounter extends DefaultRestApiCompanion[RestApiWithNeverCounter] { + final val Impl: RestApiWithNeverCounter = new RestApiWithNeverCounter { + override def neverGet: Future[Unit] = { + counter.increment() + Future.never + } + } + } +} diff --git a/rest/src/test/scala/io/udash/rest/RestTestApi.scala b/rest/src/test/scala/io/udash/rest/RestTestApi.scala index 50409fe7c..57cc8f0bd 100644 --- a/rest/src/test/scala/io/udash/rest/RestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/RestTestApi.scala @@ -1,14 +1,14 @@ package io.udash package rest -import com.avsystem.commons._ +import com.avsystem.commons.* import com.avsystem.commons.misc.{AbstractValueEnum, AbstractValueEnumCompanion, EnumCtx} import com.avsystem.commons.rpc.AsRawReal +import com.avsystem.commons.serialization.* import com.avsystem.commons.serialization.json.JsonStringOutput -import com.avsystem.commons.serialization.{GenCodec, HasPolyGenCodec, flatten, name, whenAbsent} -import io.udash.rest.openapi.adjusters._ -import io.udash.rest.openapi.{Header => OASHeader, _} -import io.udash.rest.raw._ +import io.udash.rest.openapi.adjusters.* +import io.udash.rest.openapi.{Header as OASHeader, *} +import io.udash.rest.raw.* import monix.execution.{FutureUtils, Scheduler} import scala.concurrent.Future From 016deabc9eebb8e7c351d1958a1879378e733611 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 11 Dec 2024 21:43:46 +0000 Subject: [PATCH 018/162] Update sbt-pgp to 2.3.1 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index cd30ca368..f3b54c7a6 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -11,5 +11,5 @@ addSbtPlugin("com.github.sbt" % "sbt-less" % "2.0.1") addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.10.4") // Deployment configuration -addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.0") +addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.1") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.12.2") \ No newline at end of file From d8ba9945ecc09a08ea403449786b63182f29b900 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 11 Dec 2024 21:43:49 +0000 Subject: [PATCH 019/162] Update jetty-client, jetty-rewrite, ... to 12.0.16 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 16c1e8e76..640ad810f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -28,7 +28,7 @@ object Dependencies { val scalaLoggingVersion = "3.9.5" - val jettyVersion = "12.0.15" + val jettyVersion = "12.0.16" val typesafeConfigVersion = "1.4.3" val flexmarkVersion = "0.64.8" val logbackVersion = "1.3.14" From f4cad22a7c6569cc0c9a4aafbd1a871e363fbf7c Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 11 Dec 2024 21:44:08 +0000 Subject: [PATCH 020/162] Update jetty-ee8-servlet to 12.0.16 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 16c1e8e76..640ad810f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -28,7 +28,7 @@ object Dependencies { val scalaLoggingVersion = "3.9.5" - val jettyVersion = "12.0.15" + val jettyVersion = "12.0.16" val typesafeConfigVersion = "1.4.3" val flexmarkVersion = "0.64.8" val logbackVersion = "1.3.14" From 54e2d161b8853b157c9cb1e690e9f663e769a7c7 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 11 Dec 2024 21:43:49 +0000 Subject: [PATCH 021/162] Update jetty-client, jetty-rewrite, ... to 12.0.16 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 8d6e3407b..cca3c657f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -28,7 +28,7 @@ object Dependencies { val scalaLoggingVersion = "3.9.5" - val jettyVersion = "12.0.15" + val jettyVersion = "12.0.16" val typesafeConfigVersion = "1.4.3" val flexmarkVersion = "0.64.8" val logbackVersion = "1.3.14" From 58a9fc4410e51fa614ec53aee1faaf21540f78f9 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 11 Dec 2024 21:43:46 +0000 Subject: [PATCH 022/162] Update sbt-pgp to 2.3.1 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index cd30ca368..f3b54c7a6 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -11,5 +11,5 @@ addSbtPlugin("com.github.sbt" % "sbt-less" % "2.0.1") addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.10.4") // Deployment configuration -addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.0") +addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.1") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.12.2") \ No newline at end of file From fa3a882906b1bcd8b78f19f8fb27bd9eff50acd2 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Mon, 2 Dec 2024 03:39:57 +0000 Subject: [PATCH 023/162] Update sbt, scripted-plugin to 1.10.6 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index aba0a87c5..bf2ef99c4 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ # suppress inspection "UnusedProperty" -sbt.version=1.10.5 +sbt.version=1.10.6 From 3b5972cb6e0f2a887676e59598120548ab3e26a5 Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Thu, 12 Dec 2024 12:48:37 +0100 Subject: [PATCH 024/162] Remove deprecated API usage --- .../demos/frontend/FrontendFormsTest.scala | 23 +++++++++---------- .../demos/frontend/FrontendIntroTest.scala | 18 +++++++-------- .../demos/frontend/FrontendRoutingTest.scala | 5 ++-- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendFormsTest.scala b/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendFormsTest.scala index 6af208d23..fe26be7b6 100644 --- a/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendFormsTest.scala +++ b/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendFormsTest.scala @@ -1,10 +1,9 @@ package io.udash.web.guide.demos.frontend +import com.avsystem.commons.* import io.udash.web.SeleniumTest import org.openqa.selenium.By.{ByClassName, ByCssSelector, ByTagName} -import org.openqa.selenium.{By, Keys} -import com.avsystem.commons._ import scala.util.Random class FrontendFormsTest extends SeleniumTest { @@ -23,7 +22,7 @@ class FrontendFormsTest extends SeleniumTest { el.getText == expect }) should be(true) checkboxes.findElements(new ByClassName(s"checkbox-demo-$propertyName")).asScala.forall(el => { - el.getAttribute("selected") == checkbox.getAttribute("selected") + el.getDomAttribute("selected") == checkbox.getDomAttribute("selected") }) should be(true) } } @@ -47,10 +46,10 @@ class FrontendFormsTest extends SeleniumTest { eventually { checkButtons.findElements(new ByClassName("check-buttons-demo-fruits")).asScala.forall(el => { val contains = el.getText.contains(propertyName) - if (checkbox.getAttribute("selected") != null) contains else !contains + if (checkbox.getDomAttribute("selected") != null) contains else !contains }) should be(true) checkButtons.findElements(new ByCssSelector(s"[data-label=$propertyName]")).asScala.forall(el => { - el.findElement(new ByTagName("input")).getAttribute("selected") == checkbox.getAttribute("selected") + el.findElement(new ByTagName("input")).getDomAttribute("selected") == checkbox.getDomAttribute("selected") }) should be(true) } } @@ -70,10 +69,10 @@ class FrontendFormsTest extends SeleniumTest { eventually { multiSelect.findElements(new ByClassName("multi-select-demo-fruits")).asScala.forall(el => { val contains = el.getText.contains(propertyName) - if (option.getAttribute("selected") != null) contains else !contains + if (option.getDomAttribute("selected") != null) contains else !contains }) should be(true) multiSelect.findElements(new ByTagName("select")).asScala.forall(el => { - el.findElement(new ByCssSelector(s"[value='$propertyIdx']")).getAttribute("selected") == option.getAttribute("selected") + el.findElement(new ByCssSelector(s"[value='$propertyIdx']")).getDomAttribute("selected") == option.getDomAttribute("selected") }) should be(true) } } @@ -96,8 +95,8 @@ class FrontendFormsTest extends SeleniumTest { el.getText == propertyName }) should be(true) radioButtons.findElements(new ByCssSelector(s"input")).asScala.forall(el => { - val eq = el.getAttribute("selected") == radio.getAttribute("selected") - if (el.getAttribute("value").toInt == propertyIdx) eq else !eq + val eq = el.getDomAttribute("selected") == radio.getDomAttribute("selected") + if (el.getDomAttribute("value").toInt == propertyIdx) eq else !eq }) should be(true) } } @@ -121,7 +120,7 @@ class FrontendFormsTest extends SeleniumTest { el.getText == propertyName }) should be(true) selectDemo.findElements(new ByTagName(s"select")).asScala.forall(el => { - el.findElement(new ByCssSelector(s"[value='$propertyIdx']")).getAttribute("selected") == option.getAttribute("selected") + el.findElement(new ByCssSelector(s"[value='$propertyIdx']")).getDomAttribute("selected") == option.getDomAttribute("selected") }) should be(true) } } @@ -142,7 +141,7 @@ class FrontendFormsTest extends SeleniumTest { textArea.sendKeys(text) eventually { textAreaDemo.findElements(new ByTagName(s"textarea")).asScala.forall(el => { - el.getAttribute("value") == text + el.getDomAttribute("value") == text }) should be(true) } } @@ -161,7 +160,7 @@ class FrontendFormsTest extends SeleniumTest { input.sendKeys(text) eventually { inputsDemo.findElements(new ByCssSelector(s"input[type=$tpe]")).asScala.forall(el => { - el.getAttribute("value") == text + el.getDomAttribute("value") == text }) should be(true) } } diff --git a/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendIntroTest.scala b/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendIntroTest.scala index c7569c984..94748b813 100644 --- a/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendIntroTest.scala +++ b/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendIntroTest.scala @@ -80,21 +80,21 @@ class FrontendIntroTest extends SeleniumTest { val between = demo.findElement(new ById("between")) val maximum = demo.findElement(new ById("maximum")) - var lastMinimum = minimum.getAttribute("value") - var lastBetween = between.getAttribute("value") - var lastMaximum = maximum.getAttribute("value") + var lastMinimum = minimum.getDomAttribute("value") + var lastBetween = between.getDomAttribute("value") + var lastMaximum = maximum.getDomAttribute("value") for (_ <- 1 to 5) { randomizeButton.click() eventually { - (lastMinimum != minimum.getAttribute("value") || - lastBetween != between.getAttribute("value") || - lastMaximum != maximum.getAttribute("value")) should be(true) + (lastMinimum != minimum.getDomAttribute("value") || + lastBetween != between.getDomAttribute("value") || + lastMaximum != maximum.getDomAttribute("value")) should be(true) } - lastMinimum = minimum.getAttribute("value") - lastBetween = between.getAttribute("value") - lastMaximum = maximum.getAttribute("value") + lastMinimum = minimum.getDomAttribute("value") + lastBetween = between.getDomAttribute("value") + lastMaximum = maximum.getDomAttribute("value") } } } diff --git a/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendRoutingTest.scala b/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendRoutingTest.scala index 8e7c1c447..24187bfab 100644 --- a/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendRoutingTest.scala +++ b/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendRoutingTest.scala @@ -1,7 +1,6 @@ package io.udash.web.guide.demos.frontend import io.udash.web.SeleniumTest -import org.openqa.selenium.By class FrontendRoutingTest extends SeleniumTest { override protected final val url = "/frontend/routing" @@ -51,7 +50,7 @@ class FrontendRoutingTest extends SeleniumTest { link.getText should be("/frontend/routing/pizza") } - input.getAttribute("value") should be("It should not disappear... Selenium") + input.getDomAttribute("value") should be("It should not disappear... Selenium") } "change URL basing on input without view redraw" in { @@ -81,7 +80,7 @@ class FrontendRoutingTest extends SeleniumTest { } init.getText should be("/frontend/routing") - input.getAttribute("value") should be("It should not disappear... Selenium") + input.getDomAttribute("value") should be("It should not disappear... Selenium") } } } From f949e242f9b8bbb3b44cb95f954e82fe3a1fa4eb Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Thu, 12 Dec 2024 12:48:52 +0100 Subject: [PATCH 025/162] Use JDK19+ compatible constructor for Locale --- .../io/udash/web/guide/demos/i18n/TranslationServer.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/guide/backend/src/main/scala/io/udash/web/guide/demos/i18n/TranslationServer.scala b/guide/backend/src/main/scala/io/udash/web/guide/demos/i18n/TranslationServer.scala index 1ad4a298d..e469ee391 100644 --- a/guide/backend/src/main/scala/io/udash/web/guide/demos/i18n/TranslationServer.scala +++ b/guide/backend/src/main/scala/io/udash/web/guide/demos/i18n/TranslationServer.scala @@ -1,15 +1,15 @@ package io.udash.web.guide.demos.i18n -import java.{util => ju} - -import io.udash.web.Implicits._ import io.udash.i18n.{Lang, ResourceBundlesTranslationTemplatesProvider, TranslationRPCEndpoint} +import io.udash.web.Implicits.* + +import java.util as ju class TranslationServer extends TranslationRPCEndpoint( new ResourceBundlesTranslationTemplatesProvider( TranslationServer.langs .map(lang => - Lang(lang) -> TranslationServer.bundlesNames.map(name => ju.ResourceBundle.getBundle(name, new ju.Locale(lang))) + Lang(lang) -> TranslationServer.bundlesNames.map(name => ju.ResourceBundle.getBundle(name, new ju.Locale.Builder().setLanguage(lang).build())) ).toMap ) ) From a9864f056e0ace9029475aac31a661fd065495fc Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Thu, 12 Dec 2024 13:03:34 +0100 Subject: [PATCH 026/162] Use DOM property for values --- .../demos/frontend/FrontendFormsTest.scala | 6 +++--- .../demos/frontend/FrontendIntroTest.scala | 18 +++++++++--------- .../demos/frontend/FrontendRoutingTest.scala | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendFormsTest.scala b/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendFormsTest.scala index fe26be7b6..fdfb97d07 100644 --- a/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendFormsTest.scala +++ b/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendFormsTest.scala @@ -96,7 +96,7 @@ class FrontendFormsTest extends SeleniumTest { }) should be(true) radioButtons.findElements(new ByCssSelector(s"input")).asScala.forall(el => { val eq = el.getDomAttribute("selected") == radio.getDomAttribute("selected") - if (el.getDomAttribute("value").toInt == propertyIdx) eq else !eq + if (el.getDomProperty("value").toInt == propertyIdx) eq else !eq }) should be(true) } } @@ -141,7 +141,7 @@ class FrontendFormsTest extends SeleniumTest { textArea.sendKeys(text) eventually { textAreaDemo.findElements(new ByTagName(s"textarea")).asScala.forall(el => { - el.getDomAttribute("value") == text + el.getDomProperty("value") == text }) should be(true) } } @@ -160,7 +160,7 @@ class FrontendFormsTest extends SeleniumTest { input.sendKeys(text) eventually { inputsDemo.findElements(new ByCssSelector(s"input[type=$tpe]")).asScala.forall(el => { - el.getDomAttribute("value") == text + el.getDomProperty("value") == text }) should be(true) } } diff --git a/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendIntroTest.scala b/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendIntroTest.scala index 94748b813..ec9e224bd 100644 --- a/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendIntroTest.scala +++ b/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendIntroTest.scala @@ -80,21 +80,21 @@ class FrontendIntroTest extends SeleniumTest { val between = demo.findElement(new ById("between")) val maximum = demo.findElement(new ById("maximum")) - var lastMinimum = minimum.getDomAttribute("value") - var lastBetween = between.getDomAttribute("value") - var lastMaximum = maximum.getDomAttribute("value") + var lastMinimum = minimum.getDomProperty("value") + var lastBetween = between.getDomProperty("value") + var lastMaximum = maximum.getDomProperty("value") for (_ <- 1 to 5) { randomizeButton.click() eventually { - (lastMinimum != minimum.getDomAttribute("value") || - lastBetween != between.getDomAttribute("value") || - lastMaximum != maximum.getDomAttribute("value")) should be(true) + (lastMinimum != minimum.getDomProperty("value") || + lastBetween != between.getDomProperty("value") || + lastMaximum != maximum.getDomProperty("value")) should be(true) } - lastMinimum = minimum.getDomAttribute("value") - lastBetween = between.getDomAttribute("value") - lastMaximum = maximum.getDomAttribute("value") + lastMinimum = minimum.getDomProperty("value") + lastBetween = between.getDomProperty("value") + lastMaximum = maximum.getDomProperty("value") } } } diff --git a/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendRoutingTest.scala b/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendRoutingTest.scala index 24187bfab..7f6699c97 100644 --- a/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendRoutingTest.scala +++ b/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendRoutingTest.scala @@ -50,7 +50,7 @@ class FrontendRoutingTest extends SeleniumTest { link.getText should be("/frontend/routing/pizza") } - input.getDomAttribute("value") should be("It should not disappear... Selenium") + input.getDomProperty("value") should be("It should not disappear... Selenium") } "change URL basing on input without view redraw" in { @@ -80,7 +80,7 @@ class FrontendRoutingTest extends SeleniumTest { } init.getText should be("/frontend/routing") - input.getDomAttribute("value") should be("It should not disappear... Selenium") + input.getDomProperty("value") should be("It should not disappear... Selenium") } } } From d24ee9fabf6ea230bbedd4b7e6596b047c627209 Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Fri, 13 Dec 2024 09:08:15 +0100 Subject: [PATCH 027/162] Fix selection checks --- .../scala/io/udash/web/SeleniumTest.scala | 11 ++- .../demos/frontend/FrontendFormsTest.scala | 75 +++++++++---------- 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/guide/selenium/src/test/scala/io/udash/web/SeleniumTest.scala b/guide/selenium/src/test/scala/io/udash/web/SeleniumTest.scala index b9199245e..3a711deda 100644 --- a/guide/selenium/src/test/scala/io/udash/web/SeleniumTest.scala +++ b/guide/selenium/src/test/scala/io/udash/web/SeleniumTest.scala @@ -1,14 +1,15 @@ package io.udash.web +import com.typesafe.scalalogging.StrictLogging import io.github.bonigarcia.wdm.WebDriverManager import org.openqa.selenium.firefox.{FirefoxDriver, FirefoxOptions} import org.openqa.selenium.remote.RemoteWebDriver import org.openqa.selenium.{By, Dimension, WebElement} import org.scalatest.concurrent.Eventually -import org.scalatest.time.{Millis, Seconds, Span} -import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Millis, Seconds, Span} import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Inspectors} import java.time.Duration @@ -41,13 +42,15 @@ private final class InternalServerConfig extends ServerConfig { override def createUrl(part: String): String = { require(part.startsWith("/")) - s"http://127.0.0.2:${server.port}$part" + s"http://localhost:${server.port}$part" } } -abstract class SeleniumTest extends AnyWordSpec with Matchers with BeforeAndAfterAll with BeforeAndAfterEach with Eventually { +abstract class SeleniumTest extends AnyWordSpec + with Matchers with BeforeAndAfterAll with BeforeAndAfterEach with Eventually with StrictLogging with Inspectors { override implicit val patienceConfig: PatienceConfig = PatienceConfig(scaled(Span(10, Seconds)), scaled(Span(50, Millis))) + logger.info("Setting up WebDriver") private val driverManager = WebDriverManager.firefoxdriver() driverManager.config().setServerPort(0) driverManager.setup() diff --git a/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendFormsTest.scala b/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendFormsTest.scala index fdfb97d07..6943b7209 100644 --- a/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendFormsTest.scala +++ b/guide/selenium/src/test/scala/io/udash/web/guide/demos/frontend/FrontendFormsTest.scala @@ -1,6 +1,5 @@ package io.udash.web.guide.demos.frontend -import com.avsystem.commons.* import io.udash.web.SeleniumTest import org.openqa.selenium.By.{ByClassName, ByCssSelector, ByTagName} @@ -18,12 +17,12 @@ class FrontendFormsTest extends SeleniumTest { val checkbox = checkboxes.findElement(new ByClassName(s"checkbox-demo-$propertyName")) checkbox.click() eventually { - checkboxes.findElements(new ByCssSelector(s"[data-bind=$propertyName]")).asScala.forall(el => { - el.getText == expect - }) should be(true) - checkboxes.findElements(new ByClassName(s"checkbox-demo-$propertyName")).asScala.forall(el => { - el.getDomAttribute("selected") == checkbox.getDomAttribute("selected") - }) should be(true) + forAll(checkboxes.findElements(new ByCssSelector(s"[data-bind=$propertyName]")))(el => + el.getText shouldBe expect + ) + forAll(checkboxes.findElements(new ByClassName(s"checkbox-demo-$propertyName")))(el => + el.isSelected shouldBe checkbox.isSelected + ) } } @@ -44,13 +43,13 @@ class FrontendFormsTest extends SeleniumTest { val checkbox = checkButtons.findElement(new ByCssSelector(s"[data-label=$propertyName]")).findElement(new ByTagName("input")) checkbox.click() eventually { - checkButtons.findElements(new ByClassName("check-buttons-demo-fruits")).asScala.forall(el => { + forAll(checkButtons.findElements(new ByClassName("check-buttons-demo-fruits")))(el => { val contains = el.getText.contains(propertyName) - if (checkbox.getDomAttribute("selected") != null) contains else !contains - }) should be(true) - checkButtons.findElements(new ByCssSelector(s"[data-label=$propertyName]")).asScala.forall(el => { - el.findElement(new ByTagName("input")).getDomAttribute("selected") == checkbox.getDomAttribute("selected") - }) should be(true) + assert(if (checkbox.isSelected) contains else !contains) + }) + forAll(checkButtons.findElements(new ByCssSelector(s"[data-label=$propertyName]")))(el => + el.findElement(new ByTagName("input")).isSelected shouldBe checkbox.isSelected + ) } } @@ -67,13 +66,13 @@ class FrontendFormsTest extends SeleniumTest { val option = select.findElement(new ByCssSelector(s"[value='$propertyIdx']")) option.click() eventually { - multiSelect.findElements(new ByClassName("multi-select-demo-fruits")).asScala.forall(el => { + forAll(multiSelect.findElements(new ByClassName("multi-select-demo-fruits")))(el => { val contains = el.getText.contains(propertyName) - if (option.getDomAttribute("selected") != null) contains else !contains - }) should be(true) - multiSelect.findElements(new ByTagName("select")).asScala.forall(el => { - el.findElement(new ByCssSelector(s"[value='$propertyIdx']")).getDomAttribute("selected") == option.getDomAttribute("selected") - }) should be(true) + assert(if (option.isSelected) contains else !contains) + }) + forAll(multiSelect.findElements(new ByTagName("select")))(el => { + el.findElement(new ByCssSelector(s"[value='$propertyIdx']")).isSelected shouldBe option.isSelected + }) } } @@ -91,13 +90,13 @@ class FrontendFormsTest extends SeleniumTest { val radio = radioButtons.findElement(new ByCssSelector(s"[data-label=$propertyName]")).findElement(new ByTagName("input")) driver.executeScript("arguments[0].click();", radio) eventually { - radioButtons.findElements(new ByClassName("radio-buttons-demo-fruits")).asScala.forall(el => { - el.getText == propertyName - }) should be(true) - radioButtons.findElements(new ByCssSelector(s"input")).asScala.forall(el => { - val eq = el.getDomAttribute("selected") == radio.getDomAttribute("selected") - if (el.getDomProperty("value").toInt == propertyIdx) eq else !eq - }) should be(true) + forAll(radioButtons.findElements(new ByClassName("radio-buttons-demo-fruits")))(el => + el.getText shouldBe propertyName + ) + forAll(radioButtons.findElements(new ByCssSelector(s"input")))(el => { + val eq = el.isSelected == radio.isSelected + assert(if (el.getDomProperty("value").toInt == propertyIdx) eq else !eq) + }) } } @@ -116,12 +115,12 @@ class FrontendFormsTest extends SeleniumTest { val option = select.findElement(new ByCssSelector(s"[value='$propertyIdx']")) option.click() eventually { - selectDemo.findElements(new ByClassName("select-demo-fruits")).asScala.forall(el => { - el.getText == propertyName - }) should be(true) - selectDemo.findElements(new ByTagName(s"select")).asScala.forall(el => { - el.findElement(new ByCssSelector(s"[value='$propertyIdx']")).getDomAttribute("selected") == option.getDomAttribute("selected") - }) should be(true) + forAll(selectDemo.findElements(new ByClassName("select-demo-fruits")))(el => { + el.getText shouldBe propertyName + }) + forAll(selectDemo.findElements(new ByTagName(s"select")))(el => { + el.findElement(new ByCssSelector(s"[value='$propertyIdx']")).isSelected shouldBe option.isSelected + }) } } @@ -140,9 +139,9 @@ class FrontendFormsTest extends SeleniumTest { textArea.clear() textArea.sendKeys(text) eventually { - textAreaDemo.findElements(new ByTagName(s"textarea")).asScala.forall(el => { - el.getDomProperty("value") == text - }) should be(true) + forAll(textAreaDemo.findElements(new ByTagName(s"textarea")))(el => { + el.getDomProperty("value") shouldBe text + }) } } @@ -159,9 +158,9 @@ class FrontendFormsTest extends SeleniumTest { input.clear() input.sendKeys(text) eventually { - inputsDemo.findElements(new ByCssSelector(s"input[type=$tpe]")).asScala.forall(el => { - el.getDomProperty("value") == text - }) should be(true) + forAll(inputsDemo.findElements(new ByCssSelector(s"input[type=$tpe]")))(el => { + el.getDomProperty("value") shouldBe text + }) } } From 778d3ef07f2b71a7d3b2e266cf708df8945dcda2 Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Fri, 13 Dec 2024 09:08:23 +0100 Subject: [PATCH 028/162] Restore selenium logs --- guide/selenium/src/test/resources/logback-test.xml | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 guide/selenium/src/test/resources/logback-test.xml diff --git a/guide/selenium/src/test/resources/logback-test.xml b/guide/selenium/src/test/resources/logback-test.xml deleted file mode 100644 index ea1eae09c..000000000 --- a/guide/selenium/src/test/resources/logback-test.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - \ No newline at end of file From 48ad7b0fa1d8b09fcded827b63c580db1f2cce2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Kozak?= Date: Tue, 17 Dec 2024 09:51:09 +0100 Subject: [PATCH 029/162] add test case for non-timeout cancellation --- ...eStaleJettyConnectionsOnMonixTimeout.scala | 52 +++++++++++++------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/rest/jetty/src/test/scala/io/udash/rest/jetty/CloseStaleJettyConnectionsOnMonixTimeout.scala b/rest/jetty/src/test/scala/io/udash/rest/jetty/CloseStaleJettyConnectionsOnMonixTimeout.scala index af8c764b2..9828bce93 100644 --- a/rest/jetty/src/test/scala/io/udash/rest/jetty/CloseStaleJettyConnectionsOnMonixTimeout.scala +++ b/rest/jetty/src/test/scala/io/udash/rest/jetty/CloseStaleJettyConnectionsOnMonixTimeout.scala @@ -9,24 +9,29 @@ import monix.execution.atomic.Atomic import org.eclipse.jetty.client.HttpClient import org.eclipse.jetty.ee8.servlet.{ServletContextHandler, ServletHolder} import org.eclipse.jetty.server.{NetworkConnector, Server} +import org.scalatest.BeforeAndAfterEach import org.scalatest.funsuite.AsyncFunSuite import java.net.InetSocketAddress import scala.concurrent.Future import scala.concurrent.duration.{FiniteDuration, IntMult} -final class CloseStaleJettyConnectionsOnMonixTimeout extends AsyncFunSuite { +final class CloseStaleJettyConnectionsOnMonixTimeout extends AsyncFunSuite with BeforeAndAfterEach { - test("close connection on monix task timeout") { - import monix.execution.Scheduler.Implicits.global + import monix.execution.Scheduler.Implicits.global - val MaxConnections: Int = 1 // to timeout quickly - val Connections: Int = 10 // > MaxConnections - val RequestTimeout: FiniteDuration = 1.hour // no timeout - val CallTimeout: FiniteDuration = 300.millis + private val MaxConnections: Int = 1 // to timeout quickly + private val Connections: Int = 10 // > MaxConnections + private val RequestTimeout: FiniteDuration = 1.hour // no timeout + private val CallTimeout: FiniteDuration = 300.millis + private var server: Server = _ + private var httpClient: HttpClient = _ + private var client: RestApiWithNeverCounter = _ - val server = new Server(new InetSocketAddress("localhost", 0)) { + override def beforeEach(): Unit = { + super.beforeEach() + server = new Server(new InetSocketAddress("localhost", 0)) { setHandler( new ServletContextHandler().setup( _.addServlet( @@ -40,32 +45,49 @@ final class CloseStaleJettyConnectionsOnMonixTimeout extends AsyncFunSuite { start() } - val httpClient = new HttpClient() { + httpClient = new HttpClient() { setMaxConnectionsPerDestination(MaxConnections) setIdleTimeout(RequestTimeout.toMillis) start() } - val client = JettyRestClient[RestApiWithNeverCounter]( + client = JettyRestClient[RestApiWithNeverCounter]( client = httpClient, baseUri = server.getConnectors.head |> { case connector: NetworkConnector => s"http://${connector.getHost}:${connector.getLocalPort}" }, maxResponseLength = Int.MaxValue, // to avoid unnecessary logs timeout = RequestTimeout, ) + } + override def afterEach(): Unit = { + RestApiWithNeverCounter.Impl.counter.set(0) + server.stop() + httpClient.stop() + super.afterEach() + } + + test("close connection on monix task timeout") { Task .traverse(List.range(0, Connections))(_ => Task.fromFuture(client.neverGet).timeout(CallTimeout).failed) .timeoutTo(Connections * CallTimeout + 500.millis, Task(fail("All connections should have been closed"))) // + 500 millis just in case .map(_ => assert(RestApiWithNeverCounter.Impl.counter.get() == Connections)) // neverGet should be called Connections times - .guarantee(Task { - server.stop() - httpClient.stop() - }) + .runToFuture + } + + test("close connection on monix task cancellation") { + Task + .traverse(List.range(0, Connections)) { i => + val cancelable = Task.fromFuture(client.neverGet).runAsync(_ => ()) + Task.sleep(100.millis) + .restartUntil(_ => RestApiWithNeverCounter.Impl.counter.get() >= i) + .map(_ => cancelable.cancel()) + } + .map(_ => assert(RestApiWithNeverCounter.Impl.counter.get() == Connections)) .runToFuture } } -object CloseStaleJettyConnectionsOnMonixTimeout { +private object CloseStaleJettyConnectionsOnMonixTimeout { sealed trait RestApiWithNeverCounter { final val counter = Atomic(0) @GET def neverGet: Future[Unit] From 0b2b5cdec27a855bb21248d985463ca90ed5e0ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Kozak?= Date: Tue, 17 Dec 2024 12:01:18 +0100 Subject: [PATCH 030/162] make the "close connection on monix task *" tests generic --- ...eStaleJettyConnectionsOnMonixTimeout.scala | 104 ------------------ .../udash/rest/jetty/JettyRestCallTest.scala | 4 +- .../scala/io/udash/rest/RestApiTest.scala | 38 ++++++- .../scala/io/udash/rest/RestTestApi.scala | 8 +- 4 files changed, 47 insertions(+), 107 deletions(-) delete mode 100644 rest/jetty/src/test/scala/io/udash/rest/jetty/CloseStaleJettyConnectionsOnMonixTimeout.scala diff --git a/rest/jetty/src/test/scala/io/udash/rest/jetty/CloseStaleJettyConnectionsOnMonixTimeout.scala b/rest/jetty/src/test/scala/io/udash/rest/jetty/CloseStaleJettyConnectionsOnMonixTimeout.scala deleted file mode 100644 index 9828bce93..000000000 --- a/rest/jetty/src/test/scala/io/udash/rest/jetty/CloseStaleJettyConnectionsOnMonixTimeout.scala +++ /dev/null @@ -1,104 +0,0 @@ -package io.udash.rest.jetty - -import com.avsystem.commons.misc.ScalaDurationExtensions.durationIntOps -import com.avsystem.commons.universalOps -import io.udash.rest.jetty.CloseStaleJettyConnectionsOnMonixTimeout.RestApiWithNeverCounter -import io.udash.rest.{DefaultRestApiCompanion, GET, RestServlet} -import monix.eval.Task -import monix.execution.atomic.Atomic -import org.eclipse.jetty.client.HttpClient -import org.eclipse.jetty.ee8.servlet.{ServletContextHandler, ServletHolder} -import org.eclipse.jetty.server.{NetworkConnector, Server} -import org.scalatest.BeforeAndAfterEach -import org.scalatest.funsuite.AsyncFunSuite - -import java.net.InetSocketAddress -import scala.concurrent.Future -import scala.concurrent.duration.{FiniteDuration, IntMult} - -final class CloseStaleJettyConnectionsOnMonixTimeout extends AsyncFunSuite with BeforeAndAfterEach { - - import monix.execution.Scheduler.Implicits.global - - private val MaxConnections: Int = 1 // to timeout quickly - private val Connections: Int = 10 // > MaxConnections - private val RequestTimeout: FiniteDuration = 1.hour // no timeout - private val CallTimeout: FiniteDuration = 300.millis - - private var server: Server = _ - private var httpClient: HttpClient = _ - private var client: RestApiWithNeverCounter = _ - - override def beforeEach(): Unit = { - super.beforeEach() - server = new Server(new InetSocketAddress("localhost", 0)) { - setHandler( - new ServletContextHandler().setup( - _.addServlet( - new ServletHolder( - RestServlet[RestApiWithNeverCounter](RestApiWithNeverCounter.Impl) - ), - "/*", - ) - ) - ) - start() - } - - httpClient = new HttpClient() { - setMaxConnectionsPerDestination(MaxConnections) - setIdleTimeout(RequestTimeout.toMillis) - start() - } - - client = JettyRestClient[RestApiWithNeverCounter]( - client = httpClient, - baseUri = server.getConnectors.head |> { case connector: NetworkConnector => s"http://${connector.getHost}:${connector.getLocalPort}" }, - maxResponseLength = Int.MaxValue, // to avoid unnecessary logs - timeout = RequestTimeout, - ) - } - - override def afterEach(): Unit = { - RestApiWithNeverCounter.Impl.counter.set(0) - server.stop() - httpClient.stop() - super.afterEach() - } - - test("close connection on monix task timeout") { - Task - .traverse(List.range(0, Connections))(_ => Task.fromFuture(client.neverGet).timeout(CallTimeout).failed) - .timeoutTo(Connections * CallTimeout + 500.millis, Task(fail("All connections should have been closed"))) // + 500 millis just in case - .map(_ => assert(RestApiWithNeverCounter.Impl.counter.get() == Connections)) // neverGet should be called Connections times - .runToFuture - } - - test("close connection on monix task cancellation") { - Task - .traverse(List.range(0, Connections)) { i => - val cancelable = Task.fromFuture(client.neverGet).runAsync(_ => ()) - Task.sleep(100.millis) - .restartUntil(_ => RestApiWithNeverCounter.Impl.counter.get() >= i) - .map(_ => cancelable.cancel()) - } - .map(_ => assert(RestApiWithNeverCounter.Impl.counter.get() == Connections)) - .runToFuture - } -} - -private object CloseStaleJettyConnectionsOnMonixTimeout { - sealed trait RestApiWithNeverCounter { - final val counter = Atomic(0) - @GET def neverGet: Future[Unit] - } - - object RestApiWithNeverCounter extends DefaultRestApiCompanion[RestApiWithNeverCounter] { - final val Impl: RestApiWithNeverCounter = new RestApiWithNeverCounter { - override def neverGet: Future[Unit] = { - counter.increment() - Future.never - } - } - } -} diff --git a/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala b/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala index 868ee3dcf..c32e7a5de 100644 --- a/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala +++ b/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala @@ -6,7 +6,9 @@ import io.udash.rest.{RestApiTestScenarios, ServletBasedRestApiTest} import org.eclipse.jetty.client.HttpClient final class JettyRestCallTest extends ServletBasedRestApiTest with RestApiTestScenarios { - val client: HttpClient = new HttpClient + val client: HttpClient = new HttpClient() { + setMaxConnectionsPerDestination(MaxConnections) + } def clientHandle: HandleRequest = JettyRestClient.asHandleRequest(client, s"$baseUrl/api", maxPayloadSize) diff --git a/rest/src/test/scala/io/udash/rest/RestApiTest.scala b/rest/src/test/scala/io/udash/rest/RestApiTest.scala index c41f35f10..2ca25ca12 100644 --- a/rest/src/test/scala/io/udash/rest/RestApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/RestApiTest.scala @@ -1,15 +1,25 @@ package io.udash package rest -import com.avsystem.commons._ +import com.avsystem.commons.* +import com.avsystem.commons.misc.ScalaDurationExtensions.durationIntOps import io.udash.rest.raw.RawRest import io.udash.rest.raw.RawRest.HandleRequest +import monix.eval.Task import monix.execution.Scheduler import org.scalactic.source.Position +import org.scalatest.concurrent.PatienceConfiguration.Timeout import org.scalatest.concurrent.ScalaFutures import org.scalatest.funsuite.AnyFunSuite +import scala.concurrent.duration.FiniteDuration + abstract class RestApiTest extends AnyFunSuite with ScalaFutures { + + protected final val MaxConnections: Int = 1 // to timeout quickly + protected final val Connections: Int = 10 // > MaxConnections + protected final val CallTimeout: FiniteDuration = 300.millis // << idle timeout + implicit def scheduler: Scheduler = Scheduler.global final val serverHandle: RawRest.HandleRequest = @@ -30,6 +40,9 @@ abstract class RestApiTest extends AnyFunSuite with ScalaFutures { case arr: Array[_] => IArraySeq.empty[AnyRef] ++ arr.iterator.map(mkDeep) case _ => value } + + def getNeverGetCounter(): Int = RestTestApi.Impl.neverGetCounter.get() + def resetNeverGetCounter(): Unit = RestTestApi.Impl.neverGetCounter.set(0) } trait RestApiTestScenarios extends RestApiTest { @@ -89,6 +102,29 @@ trait RestApiTestScenarios extends RestApiTest { test("body using third party type") { testCall(_.thirdPartyBody(HasThirdParty(ThirdParty(5)))) } + + test("close connection on monix task timeout") { + resetNeverGetCounter() + Task + .traverse(List.range(0, Connections))(_ => Task.deferFuture(proxy.neverGet).timeout(CallTimeout).failed) + .map(_ => assertResult(expected = Connections)(actual = getNeverGetCounter())) // neverGet should be called Connections times + .runToFuture + .futureValue(Timeout(30.seconds)) + } + + test("close connection on monix task cancellation") { + resetNeverGetCounter() + Task + .traverse(List.range(0, Connections)) { i => + val cancelable = Task.deferFuture(proxy.neverGet).runAsync(_ => ()) + Task.sleep(100.millis) + .restartUntil(_ => getNeverGetCounter() >= i) + .map(_ => cancelable.cancel()) + } + .map(_ => assertResult(expected = Connections)(actual = getNeverGetCounter())) // neverGet should be called Connections times + .runToFuture + .futureValue(Timeout(30.seconds)) + } } class DirectRestApiTest extends RestApiTestScenarios { diff --git a/rest/src/test/scala/io/udash/rest/RestTestApi.scala b/rest/src/test/scala/io/udash/rest/RestTestApi.scala index 57cc8f0bd..5fa7b489e 100644 --- a/rest/src/test/scala/io/udash/rest/RestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/RestTestApi.scala @@ -9,6 +9,7 @@ import com.avsystem.commons.serialization.json.JsonStringOutput import io.udash.rest.openapi.adjusters.* import io.udash.rest.openapi.{Header as OASHeader, *} import io.udash.rest.raw.* +import monix.execution.atomic.Atomic import monix.execution.{FutureUtils, Scheduler} import scala.concurrent.Future @@ -94,6 +95,8 @@ case class ErrorWrapper[T](error: T) object ErrorWrapper extends HasPolyGenCodec[ErrorWrapper] trait RestTestApi { + final val neverGetCounter = Atomic(0) + @GET @group("TrivialGroup") def trivialGet: Future[Unit] @GET @group("TrivialDescribedGroup") @tagDescription("something") def failingGet: Future[Unit] @GET def jsonFailingGet: Future[Unit] @@ -181,7 +184,10 @@ object RestTestApi extends DefaultRestApiCompanion[RestTestApi] { def failingGet: Future[Unit] = Future.failed(HttpErrorException.plain(503, "nie")) def jsonFailingGet: Future[Unit] = Future.failed(HttpErrorException(503, HttpBody.json(JsonValue(JsonStringOutput.write(ErrorWrapper("nie")))))) def moreFailingGet: Future[Unit] = throw HttpErrorException.plain(503, "nie") - def neverGet: Future[Unit] = Future.never + def neverGet: Future[Unit] = { + neverGetCounter.transform(_ + 1) + Future.never + } def wait(millis: Int): Future[String] = FutureUtils.delayedResult(millis.millis)(s"waited $millis ms") def getEntity(id: RestEntityId): Future[RestEntity] = Future.successful(RestEntity(id, s"${id.value}-name")) def complexGet(p1: Int, p2: String, h1: Int, h2: String, q1: Int, q2: String, q3: Opt[Int], c1: Int, c2: String): Future[RestEntity] = From a161834ed0e8dd3023556bbec553b70970d608bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Kozak?= Date: Tue, 17 Dec 2024 13:27:24 +0100 Subject: [PATCH 031/162] make tests independent --- .../test/scala/io/udash/rest/SttpRestCallTest.scala | 2 +- rest/src/test/scala/io/udash/rest/RestApiTest.scala | 11 ++++++----- rest/src/test/scala/io/udash/rest/RestTestApi.scala | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala b/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala index b3c0aad4a..b63432550 100644 --- a/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala +++ b/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala @@ -5,7 +5,7 @@ import io.udash.rest.raw.HttpErrorException import io.udash.rest.raw.RawRest.HandleRequest import sttp.client3.SttpBackend -import scala.concurrent.duration._ +import scala.concurrent.duration.* import scala.concurrent.{Await, Future} trait SttpClientRestTest extends ServletBasedRestApiTest { diff --git a/rest/src/test/scala/io/udash/rest/RestApiTest.scala b/rest/src/test/scala/io/udash/rest/RestApiTest.scala index 2ca25ca12..20e9d7502 100644 --- a/rest/src/test/scala/io/udash/rest/RestApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/RestApiTest.scala @@ -15,15 +15,16 @@ import org.scalatest.funsuite.AnyFunSuite import scala.concurrent.duration.FiniteDuration abstract class RestApiTest extends AnyFunSuite with ScalaFutures { - protected final val MaxConnections: Int = 1 // to timeout quickly protected final val Connections: Int = 10 // > MaxConnections protected final val CallTimeout: FiniteDuration = 300.millis // << idle timeout implicit def scheduler: Scheduler = Scheduler.global + private val impl: RestTestApi = RestTestApi.Impl + final val serverHandle: RawRest.HandleRequest = - RawRest.asHandleRequest[RestTestApi](RestTestApi.Impl) + RawRest.asHandleRequest[RestTestApi](impl) def clientHandle: RawRest.HandleRequest @@ -33,7 +34,7 @@ abstract class RestApiTest extends AnyFunSuite with ScalaFutures { def testCall[T](call: RestTestApi => Future[T])(implicit pos: Position): Unit = assert( call(proxy).wrapToTry.futureValue.map(mkDeep) == - call(RestTestApi.Impl).catchFailures.wrapToTry.futureValue.map(mkDeep) + call(impl).catchFailures.wrapToTry.futureValue.map(mkDeep) ) def mkDeep(value: Any): Any = value match { @@ -41,8 +42,8 @@ abstract class RestApiTest extends AnyFunSuite with ScalaFutures { case _ => value } - def getNeverGetCounter(): Int = RestTestApi.Impl.neverGetCounter.get() - def resetNeverGetCounter(): Unit = RestTestApi.Impl.neverGetCounter.set(0) + def getNeverGetCounter(): Int = impl.neverGetCounter.get() + def resetNeverGetCounter(): Unit = impl.neverGetCounter.set(0) } trait RestApiTestScenarios extends RestApiTest { diff --git a/rest/src/test/scala/io/udash/rest/RestTestApi.scala b/rest/src/test/scala/io/udash/rest/RestTestApi.scala index 5fa7b489e..2db062963 100644 --- a/rest/src/test/scala/io/udash/rest/RestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/RestTestApi.scala @@ -179,7 +179,7 @@ object RestTestApi extends DefaultRestApiCompanion[RestTestApi] { import Scheduler.Implicits.global - val Impl: RestTestApi = new RestTestApi { + def Impl: RestTestApi = new RestTestApi { def trivialGet: Future[Unit] = Future.unit def failingGet: Future[Unit] = Future.failed(HttpErrorException.plain(503, "nie")) def jsonFailingGet: Future[Unit] = Future.failed(HttpErrorException(503, HttpBody.json(JsonValue(JsonStringOutput.write(ErrorWrapper("nie")))))) From b00c43b8f31ba5de46416cda74f0d82b6121049e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Kozak?= Date: Tue, 17 Dec 2024 14:03:13 +0100 Subject: [PATCH 032/162] nits --- .../test/scala/io/udash/rest/SttpRestCallTest.scala | 2 +- rest/src/test/scala/io/udash/rest/RestApiTest.scala | 10 ++++++---- rest/src/test/scala/io/udash/rest/RestTestApi.scala | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala b/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala index b63432550..b3c0aad4a 100644 --- a/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala +++ b/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala @@ -5,7 +5,7 @@ import io.udash.rest.raw.HttpErrorException import io.udash.rest.raw.RawRest.HandleRequest import sttp.client3.SttpBackend -import scala.concurrent.duration.* +import scala.concurrent.duration._ import scala.concurrent.{Await, Future} trait SttpClientRestTest extends ServletBasedRestApiTest { diff --git a/rest/src/test/scala/io/udash/rest/RestApiTest.scala b/rest/src/test/scala/io/udash/rest/RestApiTest.scala index 20e9d7502..e19727803 100644 --- a/rest/src/test/scala/io/udash/rest/RestApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/RestApiTest.scala @@ -8,9 +8,9 @@ import io.udash.rest.raw.RawRest.HandleRequest import monix.eval.Task import monix.execution.Scheduler import org.scalactic.source.Position -import org.scalatest.concurrent.PatienceConfiguration.Timeout import org.scalatest.concurrent.ScalaFutures import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.time.{Millis, Seconds, Span} import scala.concurrent.duration.FiniteDuration @@ -21,7 +21,7 @@ abstract class RestApiTest extends AnyFunSuite with ScalaFutures { implicit def scheduler: Scheduler = Scheduler.global - private val impl: RestTestApi = RestTestApi.Impl + private val impl: RestTestApi = RestTestApi.impl() final val serverHandle: RawRest.HandleRequest = RawRest.asHandleRequest[RestTestApi](impl) @@ -47,6 +47,8 @@ abstract class RestApiTest extends AnyFunSuite with ScalaFutures { } trait RestApiTestScenarios extends RestApiTest { + override implicit val patienceConfig: PatienceConfig = PatienceConfig(scaled(Span(10, Seconds)), scaled(Span(50, Millis))) + test("trivial GET") { testCall(_.trivialGet) } @@ -110,7 +112,7 @@ trait RestApiTestScenarios extends RestApiTest { .traverse(List.range(0, Connections))(_ => Task.deferFuture(proxy.neverGet).timeout(CallTimeout).failed) .map(_ => assertResult(expected = Connections)(actual = getNeverGetCounter())) // neverGet should be called Connections times .runToFuture - .futureValue(Timeout(30.seconds)) + .futureValue } test("close connection on monix task cancellation") { @@ -124,7 +126,7 @@ trait RestApiTestScenarios extends RestApiTest { } .map(_ => assertResult(expected = Connections)(actual = getNeverGetCounter())) // neverGet should be called Connections times .runToFuture - .futureValue(Timeout(30.seconds)) + .futureValue } } diff --git a/rest/src/test/scala/io/udash/rest/RestTestApi.scala b/rest/src/test/scala/io/udash/rest/RestTestApi.scala index 2db062963..7cf330c46 100644 --- a/rest/src/test/scala/io/udash/rest/RestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/RestTestApi.scala @@ -179,7 +179,7 @@ object RestTestApi extends DefaultRestApiCompanion[RestTestApi] { import Scheduler.Implicits.global - def Impl: RestTestApi = new RestTestApi { + def impl(): RestTestApi = new RestTestApi { def trivialGet: Future[Unit] = Future.unit def failingGet: Future[Unit] = Future.failed(HttpErrorException.plain(503, "nie")) def jsonFailingGet: Future[Unit] = Future.failed(HttpErrorException(503, HttpBody.json(JsonValue(JsonStringOutput.write(ErrorWrapper("nie")))))) From 737fce1d9c243d0ae35f26f659b5aae6a9d4bc6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Kozak?= Date: Thu, 19 Dec 2024 13:29:08 +0100 Subject: [PATCH 033/162] reset counter in `beforeEach`, set IdleTimout explicitly, --- .../io/udash/rest/SttpRestCallTest.scala | 15 +++++++++--- .../udash/rest/jetty/JettyRestCallTest.scala | 1 + .../scala/io/udash/rest/RestApiTest.scala | 24 ++++++++++--------- .../scala/io/udash/rest/RestTestApi.scala | 12 ++++++---- 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala b/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala index b3c0aad4a..1c5d2d658 100644 --- a/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala +++ b/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala @@ -3,13 +3,22 @@ package rest import io.udash.rest.raw.HttpErrorException import io.udash.rest.raw.RawRest.HandleRequest -import sttp.client3.SttpBackend +import sttp.client3.{HttpClientFutureBackend, SttpBackend} -import scala.concurrent.duration._ +import java.net.http.HttpClient +import java.time.Duration as JDuration +import scala.concurrent.duration.* import scala.concurrent.{Await, Future} trait SttpClientRestTest extends ServletBasedRestApiTest { - implicit val backend: SttpBackend[Future, Any] = SttpRestClient.defaultBackend() + implicit val backend: SttpBackend[Future, Any] = HttpClientFutureBackend.usingClient( + //like defaultHttpClient but with connection timeout >> CallTimeout + HttpClient + .newBuilder() + .connectTimeout(JDuration.ofMillis(IdleTimout.toMillis)) + .followRedirects(HttpClient.Redirect.NEVER) + .build() + ) def clientHandle: HandleRequest = SttpRestClient.asHandleRequest[Future](s"$baseUrl/api") diff --git a/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala b/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala index c32e7a5de..71164072b 100644 --- a/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala +++ b/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala @@ -8,6 +8,7 @@ import org.eclipse.jetty.client.HttpClient final class JettyRestCallTest extends ServletBasedRestApiTest with RestApiTestScenarios { val client: HttpClient = new HttpClient() { setMaxConnectionsPerDestination(MaxConnections) + setIdleTimeout(IdleTimout.toMillis) } def clientHandle: HandleRequest = diff --git a/rest/src/test/scala/io/udash/rest/RestApiTest.scala b/rest/src/test/scala/io/udash/rest/RestApiTest.scala index e19727803..85d7b0045 100644 --- a/rest/src/test/scala/io/udash/rest/RestApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/RestApiTest.scala @@ -8,20 +8,27 @@ import io.udash.rest.raw.RawRest.HandleRequest import monix.eval.Task import monix.execution.Scheduler import org.scalactic.source.Position +import org.scalatest.BeforeAndAfterEach import org.scalatest.concurrent.ScalaFutures import org.scalatest.funsuite.AnyFunSuite import org.scalatest.time.{Millis, Seconds, Span} import scala.concurrent.duration.FiniteDuration -abstract class RestApiTest extends AnyFunSuite with ScalaFutures { +abstract class RestApiTest extends AnyFunSuite with ScalaFutures with BeforeAndAfterEach { + implicit def scheduler: Scheduler = Scheduler.global + protected final val MaxConnections: Int = 1 // to timeout quickly protected final val Connections: Int = 10 // > MaxConnections protected final val CallTimeout: FiniteDuration = 300.millis // << idle timeout + protected final val IdleTimout: FiniteDuration = CallTimeout * 100 - implicit def scheduler: Scheduler = Scheduler.global + protected val impl: RestTestApi.Impl = new RestTestApi.Impl - private val impl: RestTestApi = RestTestApi.impl() + override protected def beforeEach(): Unit = { + super.beforeEach() + impl.resetCounter() + } final val serverHandle: RawRest.HandleRequest = RawRest.asHandleRequest[RestTestApi](impl) @@ -41,9 +48,6 @@ abstract class RestApiTest extends AnyFunSuite with ScalaFutures { case arr: Array[_] => IArraySeq.empty[AnyRef] ++ arr.iterator.map(mkDeep) case _ => value } - - def getNeverGetCounter(): Int = impl.neverGetCounter.get() - def resetNeverGetCounter(): Unit = impl.neverGetCounter.set(0) } trait RestApiTestScenarios extends RestApiTest { @@ -107,24 +111,22 @@ trait RestApiTestScenarios extends RestApiTest { } test("close connection on monix task timeout") { - resetNeverGetCounter() Task .traverse(List.range(0, Connections))(_ => Task.deferFuture(proxy.neverGet).timeout(CallTimeout).failed) - .map(_ => assertResult(expected = Connections)(actual = getNeverGetCounter())) // neverGet should be called Connections times + .map(_ => assertResult(expected = Connections)(actual = impl.counterValue())) // neverGet should be called Connections times .runToFuture .futureValue } test("close connection on monix task cancellation") { - resetNeverGetCounter() Task .traverse(List.range(0, Connections)) { i => val cancelable = Task.deferFuture(proxy.neverGet).runAsync(_ => ()) Task.sleep(100.millis) - .restartUntil(_ => getNeverGetCounter() >= i) + .restartUntil(_ => impl.counterValue() >= i) .map(_ => cancelable.cancel()) } - .map(_ => assertResult(expected = Connections)(actual = getNeverGetCounter())) // neverGet should be called Connections times + .map(_ => assertResult(expected = Connections)(actual = impl.counterValue())) // neverGet should be called Connections times .runToFuture .futureValue } diff --git a/rest/src/test/scala/io/udash/rest/RestTestApi.scala b/rest/src/test/scala/io/udash/rest/RestTestApi.scala index 7cf330c46..64ad06b33 100644 --- a/rest/src/test/scala/io/udash/rest/RestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/RestTestApi.scala @@ -9,7 +9,7 @@ import com.avsystem.commons.serialization.json.JsonStringOutput import io.udash.rest.openapi.adjusters.* import io.udash.rest.openapi.{Header as OASHeader, *} import io.udash.rest.raw.* -import monix.execution.atomic.Atomic +import monix.execution.atomic.{Atomic, AtomicInt} import monix.execution.{FutureUtils, Scheduler} import scala.concurrent.Future @@ -95,7 +95,6 @@ case class ErrorWrapper[T](error: T) object ErrorWrapper extends HasPolyGenCodec[ErrorWrapper] trait RestTestApi { - final val neverGetCounter = Atomic(0) @GET @group("TrivialGroup") def trivialGet: Future[Unit] @GET @group("TrivialDescribedGroup") @tagDescription("something") def failingGet: Future[Unit] @@ -179,13 +178,13 @@ object RestTestApi extends DefaultRestApiCompanion[RestTestApi] { import Scheduler.Implicits.global - def impl(): RestTestApi = new RestTestApi { + final class Impl extends RestTestApi { def trivialGet: Future[Unit] = Future.unit def failingGet: Future[Unit] = Future.failed(HttpErrorException.plain(503, "nie")) def jsonFailingGet: Future[Unit] = Future.failed(HttpErrorException(503, HttpBody.json(JsonValue(JsonStringOutput.write(ErrorWrapper("nie")))))) def moreFailingGet: Future[Unit] = throw HttpErrorException.plain(503, "nie") def neverGet: Future[Unit] = { - neverGetCounter.transform(_ + 1) + counter.increment() Future.never } def wait(millis: Int): Future[String] = FutureUtils.delayedResult(millis.millis)(s"waited $millis ms") @@ -209,6 +208,11 @@ object RestTestApi extends DefaultRestApiCompanion[RestTestApi] { def wrappedBinaryEcho(bytes: Bytes): Future[Bytes] = Future.successful(bytes) def wrappedBody(id: RestEntityId): Future[RestEntityId] = Future.successful(id) def thirdPartyBody(dur: HasThirdParty): Future[HasThirdParty] = Future.successful(dur) + + /** Counter for neverGet calls */ + private val counter: AtomicInt = Atomic(0) + def counterValue(): Int = counter.get() + def resetCounter(): Unit = counter.set(0) } } From b8d59a406559346403d1a65674eb38d6cab50786 Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Mon, 23 Dec 2024 13:31:01 +0100 Subject: [PATCH 034/162] Use builder for Locale in tests --- .../ResourceBundlesTranslationTemplatesProviderTest.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/i18n/.jvm/src/test/scala/io/udash/i18n/ResourceBundlesTranslationTemplatesProviderTest.scala b/i18n/.jvm/src/test/scala/io/udash/i18n/ResourceBundlesTranslationTemplatesProviderTest.scala index f71ea8821..cf717bcec 100644 --- a/i18n/.jvm/src/test/scala/io/udash/i18n/ResourceBundlesTranslationTemplatesProviderTest.scala +++ b/i18n/.jvm/src/test/scala/io/udash/i18n/ResourceBundlesTranslationTemplatesProviderTest.scala @@ -1,13 +1,13 @@ package io.udash.i18n -import java.{util => ju} - import io.udash.testing.UdashRpcBackendTest +import java.util as ju + class ResourceBundlesTranslationTemplatesProviderTest extends UdashRpcBackendTest { val testBundlesNames = Seq("test_translations", "test2_translations") val bundles = Seq("en", "pl") - .map(lang => Lang(lang) -> testBundlesNames.map(name => ju.ResourceBundle.getBundle(name, new ju.Locale(lang)))) + .map(lang => Lang(lang) -> testBundlesNames.map(name => ju.ResourceBundle.getBundle(name, new ju.Locale.Builder().setLanguage(lang).build()))) .toMap val provider = new ResourceBundlesTranslationTemplatesProvider(bundles) @@ -39,7 +39,7 @@ class ResourceBundlesTranslationTemplatesProviderTest extends UdashRpcBackendTes } "throw an exception when mixed placeholders occurs" in { - val mixedProvider = new ResourceBundlesTranslationTemplatesProvider(Map(Lang("en") -> Seq(ju.ResourceBundle.getBundle("mixed", new ju.Locale("en"))))) + val mixedProvider = new ResourceBundlesTranslationTemplatesProvider(Map(Lang("en") -> Seq(ju.ResourceBundle.getBundle("mixed", new ju.Locale.Builder().setLanguage("en").build())))) intercept[ResourceBundlesTranslationTemplatesProvider#IndexedAndUnindexedPlaceholdersMixed]( mixedProvider.langHash(Lang("en")) should be(provider.langHash(Lang("en"))) ) From 01cf9f01028541dc0e5e504990d91146fdf776b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Kozak?= Date: Mon, 23 Dec 2024 15:06:17 +0100 Subject: [PATCH 035/162] Refactor REST API tests to use consistent async syntax Updated test syntax to use `in` for better readability and migrated to fully asynchronous assertions where applicable. Removed redundant `.futureValue` statements, ensuring non-blocking behavior across test methods. --- .../udash/rest/ServletBasedRestApiTest.scala | 5 +- .../io/udash/rest/SttpRestCallTest.scala | 21 +++++--- .../scala/io/udash/rest/RestApiTest.scala | 51 +++++++++---------- 3 files changed, 39 insertions(+), 38 deletions(-) diff --git a/rest/.jvm/src/test/scala/io/udash/rest/ServletBasedRestApiTest.scala b/rest/.jvm/src/test/scala/io/udash/rest/ServletBasedRestApiTest.scala index 284955cfe..fff02b3b2 100644 --- a/rest/.jvm/src/test/scala/io/udash/rest/ServletBasedRestApiTest.scala +++ b/rest/.jvm/src/test/scala/io/udash/rest/ServletBasedRestApiTest.scala @@ -3,12 +3,11 @@ package rest import org.eclipse.jetty.ee8.servlet.{ServletContextHandler, ServletHolder} import org.eclipse.jetty.server.Server -import org.eclipse.jetty.ee8.servlet.{ServletHandler, ServletHolder} -import scala.concurrent.duration._ +import scala.concurrent.duration.* abstract class ServletBasedRestApiTest extends RestApiTest with UsesHttpServer { - override implicit def patienceConfig: PatienceConfig = PatienceConfig(10.seconds) + override implicit val patienceConfig: PatienceConfig = PatienceConfig(10.seconds) def maxPayloadSize: Int = 1024 * 1024 def serverTimeout: FiniteDuration = 10.seconds diff --git a/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala b/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala index 1c5d2d658..f77fa0ded 100644 --- a/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala +++ b/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala @@ -30,22 +30,27 @@ trait SttpClientRestTest extends ServletBasedRestApiTest { } class SttpRestCallTest extends SttpClientRestTest with RestApiTestScenarios { - test("too large binary request") { - val future = proxy.binaryEcho(Array.fill[Byte](maxPayloadSize + 1)(5)) - val exception = future.failed.futureValue - assert(exception == HttpErrorException.plain(413, "Payload is larger than maximum 1048576 bytes (1048577)")) + "too large binary request" in { + proxy.binaryEcho(Array.fill[Byte](maxPayloadSize + 1)(5)) + .failed + .map { exception => + assert(exception == HttpErrorException.plain(413, "Payload is larger than maximum 1048576 bytes (1048577)")) + } } } class ServletTimeoutTest extends SttpClientRestTest { override def serverTimeout: FiniteDuration = 500.millis - test("rest method timeout") { - val exception = proxy.neverGet.failed.futureValue - assert(exception == HttpErrorException.plain(500, "server operation timed out after 500 milliseconds")) + "rest method timeout" in { + proxy.neverGet + .failed + .map { exception => + assert(exception == HttpErrorException.plain(500, "server operation timed out after 500 milliseconds")) + } } - test("subsequent requests with timeout") { + "subsequent requests with timeout" in { assertThrows[HttpErrorException](Await.result(proxy.wait(600), Duration.Inf)) assertThrows[HttpErrorException](Await.result(proxy.wait(600), Duration.Inf)) assertThrows[HttpErrorException](Await.result(proxy.wait(600), Duration.Inf)) diff --git a/rest/src/test/scala/io/udash/rest/RestApiTest.scala b/rest/src/test/scala/io/udash/rest/RestApiTest.scala index 85d7b0045..50142a219 100644 --- a/rest/src/test/scala/io/udash/rest/RestApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/RestApiTest.scala @@ -1,21 +1,21 @@ package io.udash package rest +import cats.implicits.catsSyntaxTuple2Semigroupal import com.avsystem.commons.* import com.avsystem.commons.misc.ScalaDurationExtensions.durationIntOps import io.udash.rest.raw.RawRest import io.udash.rest.raw.RawRest.HandleRequest +import io.udash.testing.AsyncUdashSharedTest import monix.eval.Task import monix.execution.Scheduler import org.scalactic.source.Position -import org.scalatest.BeforeAndAfterEach -import org.scalatest.concurrent.ScalaFutures -import org.scalatest.funsuite.AnyFunSuite import org.scalatest.time.{Millis, Seconds, Span} +import org.scalatest.{Assertion, BeforeAndAfterEach} import scala.concurrent.duration.FiniteDuration -abstract class RestApiTest extends AnyFunSuite with ScalaFutures with BeforeAndAfterEach { +abstract class RestApiTest extends AsyncUdashSharedTest with BeforeAndAfterEach { implicit def scheduler: Scheduler = Scheduler.global protected final val MaxConnections: Int = 1 // to timeout quickly @@ -38,11 +38,10 @@ abstract class RestApiTest extends AnyFunSuite with ScalaFutures with BeforeAndA lazy val proxy: RestTestApi = RawRest.fromHandleRequest[RestTestApi](clientHandle) - def testCall[T](call: RestTestApi => Future[T])(implicit pos: Position): Unit = - assert( - call(proxy).wrapToTry.futureValue.map(mkDeep) == - call(impl).catchFailures.wrapToTry.futureValue.map(mkDeep) - ) + def testCall[T](call: RestTestApi => Future[T])(implicit pos: Position): Future[Assertion] = + (call(proxy).wrapToTry, call(impl).catchFailures.wrapToTry).mapN { (proxyResult, implResult) => + assert(proxyResult.map(mkDeep) == implResult.map(mkDeep)) + } def mkDeep(value: Any): Any = value match { case arr: Array[_] => IArraySeq.empty[AnyRef] ++ arr.iterator.map(mkDeep) @@ -53,72 +52,71 @@ abstract class RestApiTest extends AnyFunSuite with ScalaFutures with BeforeAndA trait RestApiTestScenarios extends RestApiTest { override implicit val patienceConfig: PatienceConfig = PatienceConfig(scaled(Span(10, Seconds)), scaled(Span(50, Millis))) - test("trivial GET") { + "trivial GET" in { testCall(_.trivialGet) } - test("failing GET") { + "failing GET" in { testCall(_.failingGet) } - test("JSON failing GET") { + "JSON failing GET" in { testCall(_.jsonFailingGet) } - test("more failing GET") { + "more failing GET" in { testCall(_.moreFailingGet) } - test("complex GET") { + "complex GET" in { testCall(_.complexGet(0, "a/ +&", 1, "b/ +&", 2, "ć/ +&", Opt(3), 4, "ó /&f")) testCall(_.complexGet(0, "a/ +&", 1, "b/ +&", 2, "ć/ +&", Opt.Empty, 3, "ó /&f")) } - test("multi-param body POST") { + "multi-param body POST" in { testCall(_.multiParamPost(0, "a/ +&", 1, "b/ +&", 2, "ć/ +&", 3, "l\"l")) } - test("single body PUT") { + "single body PUT" in { testCall(_.singleBodyPut(RestEntity(RestEntityId("id"), "señor"))) } - test("form POST") { + "form POST" in { testCall(_.formPost("ó", "ą=ę", 42)) } - test("prefixed GET") { + "prefixed GET" in { testCall(_.prefix("p0", "h0", "q0").subget(0, 1, 2)) } - test("transparent prefix GET") { + "transparent prefix GET" in { testCall(_.transparentPrefix.subget(0, 1, 2)) } - test("custom response with headers") { + "custom response with headers" in { testCall(_.customResponse("walue")) } - test("binary request and response") { + "binary request and response" in { testCall(_.binaryEcho(Array.fill[Byte](5)(5))) } - test("large binary request and response") { + "large binary request and response" in { testCall(_.binaryEcho(Array.fill[Byte](1024 * 1024)(5))) } - test("body using third party type") { + "body using third party type" in { testCall(_.thirdPartyBody(HasThirdParty(ThirdParty(5)))) } - test("close connection on monix task timeout") { + "close connection on monix task timeout" in { Task .traverse(List.range(0, Connections))(_ => Task.deferFuture(proxy.neverGet).timeout(CallTimeout).failed) .map(_ => assertResult(expected = Connections)(actual = impl.counterValue())) // neverGet should be called Connections times .runToFuture - .futureValue } - test("close connection on monix task cancellation") { + "close connection on monix task cancellation" in { Task .traverse(List.range(0, Connections)) { i => val cancelable = Task.deferFuture(proxy.neverGet).runAsync(_ => ()) @@ -128,7 +126,6 @@ trait RestApiTestScenarios extends RestApiTest { } .map(_ => assertResult(expected = Connections)(actual = impl.counterValue())) // neverGet should be called Connections times .runToFuture - .futureValue } } From c1492564a3d178f43cd0689ad56f772843c75ce1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Kozak?= Date: Mon, 23 Dec 2024 15:45:02 +0100 Subject: [PATCH 036/162] Add detailed comments for custom HttpClient setups Clarify the purpose of HttpClient configurations in `SttpRestCallTest` and `JettyRestCallTest`. The added comments explain the use of a connection timeout significantly exceeding the CallTimeout value, improving code readability and maintainability. --- .../.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala | 5 ++++- .../test/scala/io/udash/rest/jetty/JettyRestCallTest.scala | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala b/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala index f77fa0ded..38fdee737 100644 --- a/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala +++ b/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala @@ -11,8 +11,11 @@ import scala.concurrent.duration.* import scala.concurrent.{Await, Future} trait SttpClientRestTest extends ServletBasedRestApiTest { + /** + * Similar to the defaultHttpClient, but with a connection timeout + * significantly exceeding the value of the CallTimeout + */ implicit val backend: SttpBackend[Future, Any] = HttpClientFutureBackend.usingClient( - //like defaultHttpClient but with connection timeout >> CallTimeout HttpClient .newBuilder() .connectTimeout(JDuration.ofMillis(IdleTimout.toMillis)) diff --git a/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala b/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala index 71164072b..22fb5ce69 100644 --- a/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala +++ b/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala @@ -6,6 +6,10 @@ import io.udash.rest.{RestApiTestScenarios, ServletBasedRestApiTest} import org.eclipse.jetty.client.HttpClient final class JettyRestCallTest extends ServletBasedRestApiTest with RestApiTestScenarios { + /** + * Similar to the default HttpClient, but with a connection timeout + * significantly exceeding the value of the CallTimeout + */ val client: HttpClient = new HttpClient() { setMaxConnectionsPerDestination(MaxConnections) setIdleTimeout(IdleTimout.toMillis) From b0129ba5df28d6320572ab0a2544baad7c12b45e Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Thu, 26 Dec 2024 00:14:51 +0000 Subject: [PATCH 037/162] Update sbt, scripted-plugin to 1.10.7 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index bf2ef99c4..1dd00a2ca 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ # suppress inspection "UnusedProperty" -sbt.version=1.10.6 +sbt.version=1.10.7 From 7c28c57d744d09bd150d592652eec96765b3be1c Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 31 Dec 2024 18:53:26 +0000 Subject: [PATCH 038/162] Update logback-classic to 1.3.15 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index cca3c657f..812f387e1 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -31,7 +31,7 @@ object Dependencies { val jettyVersion = "12.0.16" val typesafeConfigVersion = "1.4.3" val flexmarkVersion = "0.64.8" - val logbackVersion = "1.3.14" + val logbackVersion = "1.3.15" val janinoVersion = "3.1.12" val fontAwesomeVersion = "5.10.1" val svg4everybodyVersion = "2.1.9" From 578e6dedbc22cd22ddbc495bf37fad33ba0ad0a3 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 31 Dec 2024 18:53:37 +0000 Subject: [PATCH 039/162] Update monix to 3.10.2 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index cca3c657f..ab3b213ed 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -24,7 +24,7 @@ object Dependencies { val circeDerivationVersion = "0.13.0-M5" // Tests only val monixVersion = "3.4.1" // udash-rest only - val sttpVersion = "3.10.1" + val sttpVersion = "3.10.2" val scalaLoggingVersion = "3.9.5" From 966280a6caddc82a95960118a6d0a2cffa4314a6 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 8 Jan 2025 19:39:45 +0000 Subject: [PATCH 040/162] Update sbt-native-packager to 1.11.0 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index f3b54c7a6..be3f33614 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -8,7 +8,7 @@ addSbtPlugin("org.scala-js" % "sbt-jsdependencies" % "1.0.2") addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.4") addSbtPlugin("org.jetbrains.scala" % "sbt-ide-settings" % "1.1.2") addSbtPlugin("com.github.sbt" % "sbt-less" % "2.0.1") -addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.10.4") +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.0") // Deployment configuration addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.1") From c14872f29608c7b66b16aaef7516f77e8ea0ae29 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sun, 12 Jan 2025 21:53:41 +0000 Subject: [PATCH 041/162] Update upickle to 4.1.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index acf9faec1..e026ab2ae 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -19,7 +19,7 @@ object Dependencies { val atmosphereJSVersion = "3.1.3" val atmosphereVersion = "2.7.15" - val upickleVersion = "4.0.2" // Tests only + val upickleVersion = "4.1.0" // Tests only val circeVersion = "0.14.10" // Tests only val circeDerivationVersion = "0.13.0-M5" // Tests only val monixVersion = "3.4.1" // udash-rest only From dd3e8ea159821bb85d54e4be357def28b3c0be87 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sun, 12 Jan 2025 21:53:51 +0000 Subject: [PATCH 042/162] Update sbt-scalajs, scalajs-compiler, ... to 1.18.1 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index f3b54c7a6..f4e998d49 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -3,7 +3,7 @@ logLevel := Level.Warn libraryDependencies += "org.scala-js" %% "scalajs-env-selenium" % "1.1.1" libraryDependencies += "org.scala-js" %% "scalajs-env-jsdom-nodejs" % "1.1.0" -addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.17.0") +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.18.1") addSbtPlugin("org.scala-js" % "sbt-jsdependencies" % "1.0.2") addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.4") addSbtPlugin("org.jetbrains.scala" % "sbt-ide-settings" % "1.1.2") From 2449c3af737f81c2be4135f7f62236b92f7c592e Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Tue, 14 Jan 2025 10:35:09 +0100 Subject: [PATCH 043/162] AVSystem Commons 2.21.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index e026ab2ae..9c8095f4d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -14,7 +14,7 @@ object Dependencies { val scalaCssVersion = "1.0.0" val servletVersion = "4.0.1" - val avsCommonsVersion = "2.20.0" + val avsCommonsVersion = "2.21.0" val atmosphereJSVersion = "3.1.3" val atmosphereVersion = "2.7.15" From ae61d7eb09677b308d70fad05bfc7dde11695786 Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Tue, 14 Jan 2025 11:36:48 +0100 Subject: [PATCH 044/162] Fix docker publishing --- .github/workflows/docker.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 318679634..ee4997ab6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -16,16 +16,17 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v3 - - uses: docker/login-action@v2 + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: udashframework password: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: temurin - java-version: 11 + java-version: 17 + cache: sbt - name: Get version id: get_tag_name run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT From 342e69741f257b6563144e0435b1ea8374ef2950 Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Tue, 14 Jan 2025 11:39:44 +0100 Subject: [PATCH 045/162] Fix docker publishing once again - ubuntu version --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ee4997ab6..788fd351e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,7 +9,7 @@ env: jobs: build-and-push-image: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md # only run on tag push if: github.event_name == 'push' && (startsWith(github.ref, 'refs/tags/v')) permissions: From 3780f01031bbffc70d149c82423eeb04f60a09fd Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sun, 19 Jan 2025 12:56:45 +0000 Subject: [PATCH 046/162] Update scala-library, scala-reflect to 2.13.16 --- .github/workflows/ci.yml | 2 +- project/Dependencies.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f7ebe932..534e2cea8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - scala: [ 2.13.15 ] + scala: [ 2.13.16 ] command: [ udash-jvm/test, udash-js/test, guide-selenium/test ] steps: - uses: actions/checkout@v4 diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 9c8095f4d..5e9a531c5 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -5,7 +5,7 @@ import sbt.* import sbt.Keys.scalaVersion object Dependencies { - val versionOfScala = "2.13.15" //update .github/workflows/ci.yml as well + val versionOfScala = "2.13.16" //update .github/workflows/ci.yml as well val jqueryWrapperVersion = "3.3.0" From da6a9231139d2437e4f2ea1431d14419ec253e65 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 25 Jan 2025 06:49:34 +0000 Subject: [PATCH 047/162] Update sbt-scalajs, scalajs-compiler, ... to 1.18.2 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 1ad90d608..dc80d50ec 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -3,7 +3,7 @@ logLevel := Level.Warn libraryDependencies += "org.scala-js" %% "scalajs-env-selenium" % "1.1.1" libraryDependencies += "org.scala-js" %% "scalajs-env-jsdom-nodejs" % "1.1.0" -addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.18.1") +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.18.2") addSbtPlugin("org.scala-js" % "sbt-jsdependencies" % "1.0.2") addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.4") addSbtPlugin("org.jetbrains.scala" % "sbt-ide-settings" % "1.1.2") From ee43b7a79cd5be813fc8e2ca655fc76fb19b668d Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 25 Jan 2025 06:49:43 +0000 Subject: [PATCH 048/162] Update selenium-java to 4.28.1 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 9c8095f4d..95d1e53a3 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -42,7 +42,7 @@ object Dependencies { val bootstrap4DatepickerVersion = "5.39.0" val momentJsVersion = "2.30.1" - val seleniumVersion = "4.27.0" + val seleniumVersion = "4.28.1" val webDriverManagerVersion = "5.9.2" val scalaJsBenchmarkVersion = "0.10.0" From a02aafa0e956a6b9c12f642fe1a0d753b3cfc5cd Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sun, 19 Jan 2025 12:56:45 +0000 Subject: [PATCH 049/162] Update scala-library, scala-reflect to 2.13.16 --- .github/workflows/ci.yml | 2 +- project/Dependencies.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f7ebe932..534e2cea8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - scala: [ 2.13.15 ] + scala: [ 2.13.16 ] command: [ udash-jvm/test, udash-js/test, guide-selenium/test ] steps: - uses: actions/checkout@v4 diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 9c8095f4d..5e9a531c5 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -5,7 +5,7 @@ import sbt.* import sbt.Keys.scalaVersion object Dependencies { - val versionOfScala = "2.13.15" //update .github/workflows/ci.yml as well + val versionOfScala = "2.13.16" //update .github/workflows/ci.yml as well val jqueryWrapperVersion = "3.3.0" From e0bf0138971e69718da0da7d75820349256c8d56 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 25 Jan 2025 06:49:43 +0000 Subject: [PATCH 050/162] Update selenium-java to 4.28.1 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 5e9a531c5..6a3c7800c 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -42,7 +42,7 @@ object Dependencies { val bootstrap4DatepickerVersion = "5.39.0" val momentJsVersion = "2.30.1" - val seleniumVersion = "4.27.0" + val seleniumVersion = "4.28.1" val webDriverManagerVersion = "5.9.2" val scalaJsBenchmarkVersion = "0.10.0" From f4214148e126128046a215dd1e03e505c8640227 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 5 Feb 2025 21:38:31 +0000 Subject: [PATCH 051/162] Update sbt-native-packager to 1.11.1 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index dc80d50ec..07fe5c046 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -8,7 +8,7 @@ addSbtPlugin("org.scala-js" % "sbt-jsdependencies" % "1.0.2") addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.4") addSbtPlugin("org.jetbrains.scala" % "sbt-ide-settings" % "1.1.2") addSbtPlugin("com.github.sbt" % "sbt-less" % "2.0.1") -addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.0") +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.1") // Deployment configuration addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.1") From 892d489df1c1fba5b36122a1ad7d5c825e48b8a4 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 5 Feb 2025 21:38:34 +0000 Subject: [PATCH 052/162] Update monix to 3.10.3 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6a3c7800c..7b6dd988d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -24,7 +24,7 @@ object Dependencies { val circeDerivationVersion = "0.13.0-M5" // Tests only val monixVersion = "3.4.1" // udash-rest only - val sttpVersion = "3.10.2" + val sttpVersion = "3.10.3" val scalaLoggingVersion = "3.9.5" From 128e20fb4e24fab4537bb657ff6b05fa4619cb32 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 15 Feb 2025 17:06:46 +0000 Subject: [PATCH 053/162] Update webdrivermanager to 5.9.3 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 7b6dd988d..5dde648b4 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -43,7 +43,7 @@ object Dependencies { val momentJsVersion = "2.30.1" val seleniumVersion = "4.28.1" - val webDriverManagerVersion = "5.9.2" + val webDriverManagerVersion = "5.9.3" val scalaJsBenchmarkVersion = "0.10.0" val compilerPlugins = Def.setting(Seq( From d29c2fbb28674073a1742f16765485f074d4c5cc Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Fri, 21 Feb 2025 16:04:51 +0000 Subject: [PATCH 054/162] Update selenium-java to 4.29.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 7b6dd988d..d266e3486 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -42,7 +42,7 @@ object Dependencies { val bootstrap4DatepickerVersion = "5.39.0" val momentJsVersion = "2.30.1" - val seleniumVersion = "4.28.1" + val seleniumVersion = "4.29.0" val webDriverManagerVersion = "5.9.2" val scalaJsBenchmarkVersion = "0.10.0" From a55944fada1fcf891feacedbe3f96b3698e945ec Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 18 Mar 2025 01:56:19 +0000 Subject: [PATCH 055/162] Update circe-core, circe-parser to 0.14.12 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 961f6f533..60bac4c3a 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -20,7 +20,7 @@ object Dependencies { val atmosphereVersion = "2.7.15" val upickleVersion = "4.1.0" // Tests only - val circeVersion = "0.14.10" // Tests only + val circeVersion = "0.14.12" // Tests only val circeDerivationVersion = "0.13.0-M5" // Tests only val monixVersion = "3.4.1" // udash-rest only From e634e4af698c85cc7dd6744c8afc20e744be13d8 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 18 Mar 2025 01:56:21 +0000 Subject: [PATCH 056/162] Update jetty-client, jetty-rewrite, ... to 12.0.18 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 961f6f533..4b9c07455 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -28,7 +28,7 @@ object Dependencies { val scalaLoggingVersion = "3.9.5" - val jettyVersion = "12.0.16" + val jettyVersion = "12.0.18" val typesafeConfigVersion = "1.4.3" val flexmarkVersion = "0.64.8" val logbackVersion = "1.3.15" From 33df2e0fd670d35211af573e87c76244ef14fd51 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 18 Mar 2025 01:56:44 +0000 Subject: [PATCH 057/162] Update sbt, scripted-plugin to 1.10.11 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index 1dd00a2ca..d9109d9ee 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ # suppress inspection "UnusedProperty" -sbt.version=1.10.7 +sbt.version=1.10.11 From 0297bc566b1c5e8c94c6e8b9af7803562c69c0ac Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Thu, 20 Mar 2025 04:48:25 +0000 Subject: [PATCH 058/162] Update webdrivermanager to 6.0.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 961f6f533..e5574a37f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -43,7 +43,7 @@ object Dependencies { val momentJsVersion = "2.30.1" val seleniumVersion = "4.29.0" - val webDriverManagerVersion = "5.9.3" + val webDriverManagerVersion = "6.0.0" val scalaJsBenchmarkVersion = "0.10.0" val compilerPlugins = Def.setting(Seq( From 8c68173510069abf91aa806687f925424586bd74 Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Thu, 27 Mar 2025 16:22:08 +0100 Subject: [PATCH 059/162] Streaming support in Udash REST --- project/Dependencies.scala | 2 +- .../scala/io/udash/rest/RestServlet.scala | 132 ++++++--- .../test/resources/StreamingRestTestApi.json | 126 +++++++++ .../udash/rest/ServletBasedRestApiTest.scala | 5 +- .../test/scala/io/udash/rest/SomeApi.scala | 26 -- .../io/udash/rest/SttpRestCallTest.scala | 5 +- .../io/udash/rest/examples/GenericApi.scala | 2 +- .../rest/openapi/OpenApiGenerationTest.scala | 12 +- .../io/udash/rest/jetty/JettyRestClient.scala | 266 ++++++++++++++---- .../udash/rest/jetty/JettyRestCallTest.scala | 13 +- .../scala/io/udash/rest/annotations.scala | 18 +- .../udash/rest/openapi/OpenApiMetadata.scala | 6 +- .../io/udash/rest/openapi/RestSchema.scala | 18 +- .../scala/io/udash/rest/raw/HttpBody.scala | 5 +- .../scala/io/udash/rest/raw/RawRest.scala | 172 +++++++++-- .../io/udash/rest/raw/RestMetadata.scala | 42 ++- .../io/udash/rest/raw/RestResponse.scala | 153 +++++++++- .../io/udash/rest/raw/StreamedBody.scala | 99 +++++++ .../main/scala/io/udash/rest/util/Utils.scala | 16 ++ .../io/udash/rest/DirectRestApiTest.scala | 17 ++ .../scala/io/udash/rest/RestApiTest.scala | 32 ++- .../io/udash/rest/StreamingRestTestApi.scala | 25 ++ .../scala/io/udash/rest/raw/RawRestTest.scala | 2 + .../io/udash/rest/raw/ServerImplApiTest.scala | 2 + 24 files changed, 1019 insertions(+), 177 deletions(-) create mode 100644 rest/.jvm/src/test/resources/StreamingRestTestApi.json delete mode 100644 rest/.jvm/src/test/scala/io/udash/rest/SomeApi.scala create mode 100644 rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala create mode 100644 rest/src/main/scala/io/udash/rest/util/Utils.scala create mode 100644 rest/src/test/scala/io/udash/rest/DirectRestApiTest.scala create mode 100644 rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 961f6f533..7550d1f26 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -114,7 +114,7 @@ object Dependencies { "javax.servlet" % "javax.servlet-api" % servletVersion, "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion, "org.eclipse.jetty" % "jetty-server" % jettyVersion % Test, - "org.eclipse.jetty.ee8" % "jetty-ee8-servlet" % jettyVersion % Test + "org.eclipse.jetty.ee8" % "jetty-ee8-servlet" % jettyVersion % Test, )) val restSjsDeps = restCrossDeps diff --git a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala index d09c73086..39433d6c8 100644 --- a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala +++ b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala @@ -9,6 +9,7 @@ import io.udash.rest.raw._ import io.udash.utils.URLEncoder import monix.eval.Task import monix.execution.Scheduler +import monix.reactive.Consumer import java.io.ByteArrayOutputStream import java.util.concurrent.atomic.AtomicBoolean @@ -21,6 +22,7 @@ object RestServlet { final val DefaultHandleTimeout = 30.seconds final val DefaultMaxPayloadSize = 16 * 1024 * 1024L // 16MB final val CookieHeader = "Cookie" + private final val BufferSize = 8192 /** * Wraps an implementation of some REST API trait into a Java Servlet. @@ -33,18 +35,17 @@ object RestServlet { @explicitGenerics def apply[RestApi: RawRest.AsRawRpc : RestMetadata]( apiImpl: RestApi, handleTimeout: FiniteDuration = DefaultHandleTimeout, - maxPayloadSize: Long = DefaultMaxPayloadSize + maxPayloadSize: Long = DefaultMaxPayloadSize, )(implicit scheduler: Scheduler - ): RestServlet = new RestServlet(RawRest.asHandleRequest[RestApi](apiImpl), handleTimeout, maxPayloadSize) - - private final val BufferSize = 8192 + ): RestServlet = + new RestServlet(RawRest.asHandleRequestWithStreaming[RestApi](apiImpl), handleTimeout, maxPayloadSize) } class RestServlet( - handleRequest: RawRest.HandleRequest, + handleRequest: RawRest.HandleRequestWithStreaming, handleTimeout: FiniteDuration = DefaultHandleTimeout, - maxPayloadSize: Long = DefaultMaxPayloadSize + maxPayloadSize: Long = DefaultMaxPayloadSize, )(implicit scheduler: Scheduler ) extends HttpServlet with LazyLogging { @@ -66,9 +67,12 @@ class RestServlet( // readRequest must execute in Jetty thread but we want exceptions to be handled uniformly, hence the Try val udashRequest = Try(readRequest(request)) - val cancelable = Task.defer(handleRequest(udashRequest.get)).executeAsync.runAsync { - case Right(restResponse) => - completeWith(writeResponse(response, restResponse)) + val cancelable = Task.defer(handleRequest(udashRequest.get)).flatMap { rr => + Task(setResponseHeaders(response, rr.code, rr.headers)) >> + writeResponseBody(response, rr) + }.executeAsync.runAsync { + case Right(_) => + asyncContext.complete() case Left(e: HttpErrorException) => completeWith(writeResponse(response, e.toResponse)) case Left(e) => @@ -88,6 +92,92 @@ class RestServlet( }) } + private def setResponseHeaders(response: HttpServletResponse, code: Int, headers: IMapping[PlainValue]): Unit = { + response.setStatus(code) + headers.entries.foreach { + case (name, PlainValue(value)) => response.addHeader(name, value) + } + } + + private def writeNonEmptyBody(response: HttpServletResponse, body: HttpBody.NonEmpty): Unit = { + val bytes = body.bytes + response.setContentType(body.contentType) + response.setContentLength(bytes.length) + response.getOutputStream.write(bytes) + } + + private def writeNonEmptyStreamedBody( + response: HttpServletResponse, + stream: StreamedRestResponse, + body: StreamedBody.NonEmpty, + ): Task[Unit] = Task.defer { + body match { + case single: StreamedBody.Single => + Task.eval(writeNonEmptyBody(response, single.body)) + case binary: StreamedBody.RawBinary => + // TODO streaming document no content length behaviour in relation to the client + response.setContentType(binary.contentType) + binary.content.bufferTumbling(stream.batchSize).consumeWith(Consumer.foreach { batch => + batch.foreach(e => response.getOutputStream.write(e)) + response.getOutputStream.flush() + }) + case jsonList: StreamedBody.JsonList => + // TODO streaming document no content length behaviour in relation to the client + response.setContentType(jsonList.contentType) + response.getOutputStream.write("[".getBytes(jsonList.charset)) + jsonList.elements + .bufferTumbling(stream.batchSize) + .zipWithIndex + .consumeWith(Consumer.foreach { case (batch, idx) => + val firstBatch = idx == 0 + if (firstBatch) + batch.iterator.zipWithIndex.foreach { case (e, idx) => + if (idx != 0) { + response.getOutputStream.write(",".getBytes(jsonList.charset)) + } + response.getOutputStream.write(e.value.getBytes(jsonList.charset)) + } + else + batch.foreach { e => + response.getOutputStream.write(",".getBytes(jsonList.charset)) + response.getOutputStream.write(e.value.getBytes(jsonList.charset)) + } + response.getOutputStream.flush() + }) + .map(_ => response.getOutputStream.write("]".getBytes(jsonList.charset))) + } + } + + private def writeResponseBody(response: HttpServletResponse, rr: AbstractRestResponse): Task[Unit] = + rr match { + case resp: RestResponse => + resp.body match { + case HttpBody.Empty => Task.unit + case neBody: HttpBody.NonEmpty => Task(writeNonEmptyBody(response, neBody)) + } + case stream: StreamedRestResponse => + stream.body match { + case StreamedBody.Empty => Task.unit + case neBody: StreamedBody.NonEmpty => writeNonEmptyStreamedBody(response, stream, neBody) + } + } + + private def writeResponse(response: HttpServletResponse, restResponse: RestResponse): Unit = { + setResponseHeaders(response, restResponse.code, restResponse.headers) + restResponse.body match { + case HttpBody.Empty => + case neBody: HttpBody.NonEmpty => writeNonEmptyBody(response, neBody) + } + } + + private def writeFailure(response: HttpServletResponse, message: Opt[String]): Unit = { + response.setStatus(500) + message.foreach { msg => + response.setContentType(s"text/plain;charset=utf-8") + response.getWriter.write(msg) + } + } + private def readParameters(request: HttpServletRequest): RestParameters = { // can't use request.getPathInfo because it decodes the URL before we can split it val pathPrefix = request.getContextPath.orEmpty + request.getServletPath.orEmpty @@ -162,28 +252,4 @@ class RestServlet( val body = readBody(request) RestRequest(method, parameters, body) } - - private def writeResponse(response: HttpServletResponse, restResponse: RestResponse): Unit = { - response.setStatus(restResponse.code) - restResponse.headers.entries.foreach { - case (name, PlainValue(value)) => response.addHeader(name, value) - } - restResponse.body match { - case HttpBody.Empty => - case neBody: HttpBody.NonEmpty => - // TODO: can we improve performance by avoiding intermediate byte array for textual content? - val bytes = neBody.bytes - response.setContentType(neBody.contentType) - response.setContentLength(bytes.length) - response.getOutputStream.write(bytes) - } - } - - private def writeFailure(response: HttpServletResponse, message: Opt[String]): Unit = { - response.setStatus(500) - message.foreach { msg => - response.setContentType(s"text/plain;charset=utf-8") - response.getWriter.write(msg) - } - } } diff --git a/rest/.jvm/src/test/resources/StreamingRestTestApi.json b/rest/.jvm/src/test/resources/StreamingRestTestApi.json new file mode 100644 index 000000000..39cdb5407 --- /dev/null +++ b/rest/.jvm/src/test/resources/StreamingRestTestApi.json @@ -0,0 +1,126 @@ +{ + "openapi": "3.0.2", + "info": { + "title": "Streaming Test API", + "version": "0.1", + "description": "Some test REST API" + }, + "paths": { + "/jsonStream": { + "get": { + "operationId": "jsonStream", + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/RestEntity" + } + } + } + } + } + } + } + }, + "/simpleStream": { + "get": { + "operationId": "simpleStream", + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "servers": [ + { + "url": "http://localhost" + } + ], + "components": { + "schemas": { + "RestEntity": { + "type": "object", + "description": "REST entity", + "properties": { + "id": { + "description": "entity id", + "allOf": [ + { + "$ref": "#/components/schemas/RestEntityId" + } + ] + }, + "name": { + "type": "string", + "default": "anonymous" + }, + "subentity": { + "description": "recursive optional subentity", + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/RestEntity" + } + ], + "default": null + }, + "enumField": { + "allOf": [ + { + "$ref": "#/components/schemas/RestEntityEnumCustom" + } + ], + "default": "OptionOne" + }, + "inlinedEnumField": { + "type": "string", + "enum": [ + "Option1", + "Option2" + ], + "default": "Option1" + }, + "enumMap": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/RestEntityEnumCustom" + }, + "default": {} + } + }, + "required": [ + "id" + ] + }, + "RestEntityEnumCustom": { + "type": "string", + "description": "Example named enum", + "enum": [ + "OptionOne", + "OptionTwo" + ], + "example": "OptionOne" + }, + "RestEntityId": { + "type": "string", + "description": "Entity identifier" + } + } + } +} \ No newline at end of file diff --git a/rest/.jvm/src/test/scala/io/udash/rest/ServletBasedRestApiTest.scala b/rest/.jvm/src/test/scala/io/udash/rest/ServletBasedRestApiTest.scala index fff02b3b2..8ced322fb 100644 --- a/rest/.jvm/src/test/scala/io/udash/rest/ServletBasedRestApiTest.scala +++ b/rest/.jvm/src/test/scala/io/udash/rest/ServletBasedRestApiTest.scala @@ -14,9 +14,10 @@ abstract class ServletBasedRestApiTest extends RestApiTest with UsesHttpServer { protected def setupServer(server: Server): Unit = { val servlet = new RestServlet(serverHandle, serverTimeout, maxPayloadSize) - val holder = new ServletHolder(servlet) + val streamingServlet = new RestServlet(streamingServerHandle, serverTimeout, maxPayloadSize) val handler = new ServletContextHandler() - handler.addServlet(holder, "/api/*") + handler.addServlet(new ServletHolder(servlet), "/api/*") + handler.addServlet(new ServletHolder(streamingServlet), "/stream-api/*") server.setHandler(handler) } } diff --git a/rest/.jvm/src/test/scala/io/udash/rest/SomeApi.scala b/rest/.jvm/src/test/scala/io/udash/rest/SomeApi.scala deleted file mode 100644 index b9e748ed6..000000000 --- a/rest/.jvm/src/test/scala/io/udash/rest/SomeApi.scala +++ /dev/null @@ -1,26 +0,0 @@ -package io.udash -package rest - -import scala.concurrent.Future - -trait SomeApi { - @GET - def hello(who: String): Future[String] - - @POST("hello") - def helloThere(who: String): Future[String] -} - -object SomeApi extends DefaultRestApiCompanion[SomeApi] { - def format(who: String) = s"Hello, $who!" - val poison: String = "poison" - - val impl: SomeApi = new SomeApi { - override def hello(who: String): Future[String] = { - if (who == poison) throw new IllegalArgumentException(poison) - else Future.successful(format(who)) - } - - override def helloThere(who: String): Future[String] = hello(who) - } -} diff --git a/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala b/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala index 38fdee737..86d456f9a 100644 --- a/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala +++ b/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala @@ -1,8 +1,7 @@ package io.udash package rest -import io.udash.rest.raw.HttpErrorException -import io.udash.rest.raw.RawRest.HandleRequest +import io.udash.rest.raw.{HttpErrorException, RawRest} import sttp.client3.{HttpClientFutureBackend, SttpBackend} import java.net.http.HttpClient @@ -23,7 +22,7 @@ trait SttpClientRestTest extends ServletBasedRestApiTest { .build() ) - def clientHandle: HandleRequest = + def clientHandle: RawRest.HandleRequest = SttpRestClient.asHandleRequest[Future](s"$baseUrl/api") override protected def afterAll(): Unit = { diff --git a/rest/.jvm/src/test/scala/io/udash/rest/examples/GenericApi.scala b/rest/.jvm/src/test/scala/io/udash/rest/examples/GenericApi.scala index 1cc3cccdd..09dd065c8 100644 --- a/rest/.jvm/src/test/scala/io/udash/rest/examples/GenericApi.scala +++ b/rest/.jvm/src/test/scala/io/udash/rest/examples/GenericApi.scala @@ -17,4 +17,4 @@ object GenericApi { import openapi._ implicit def openApiMetadata[T: RestSchema]: OpenApiMetadata[GenericApi[T]] = OpenApiMetadata.materialize -} \ No newline at end of file +} diff --git a/rest/.jvm/src/test/scala/io/udash/rest/openapi/OpenApiGenerationTest.scala b/rest/.jvm/src/test/scala/io/udash/rest/openapi/OpenApiGenerationTest.scala index 6ca459010..fc6949021 100644 --- a/rest/.jvm/src/test/scala/io/udash/rest/openapi/OpenApiGenerationTest.scala +++ b/rest/.jvm/src/test/scala/io/udash/rest/openapi/OpenApiGenerationTest.scala @@ -2,7 +2,7 @@ package io.udash package rest.openapi import com.avsystem.commons.serialization.json.JsonStringOutput -import io.udash.rest.RestTestApi +import io.udash.rest.{RestTestApi, StreamingRestTestApi} import scala.io.Source import org.scalatest.funsuite.AnyFunSuite @@ -17,4 +17,14 @@ class OpenApiGenerationTest extends AnyFunSuite { val actual = JsonStringOutput.writePretty(openapi) assert(actual == expected) } + + test("openapi for StreamingRestTestApi") { + val openapi = StreamingRestTestApi.openapiMetadata.openapi( + Info("Streaming Test API", "0.1", description = "Some test REST API"), + servers = List(Server("http://localhost")) + ) + val expected = Source.fromInputStream(getClass.getResourceAsStream("/StreamingRestTestApi.json")).getLines().mkString("\n") + val actual = JsonStringOutput.writePretty(openapi) + assert(actual == expected) + } } diff --git a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala index 86f1118a1..5b81b815d 100644 --- a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala +++ b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala @@ -3,10 +3,13 @@ package rest.jetty import com.avsystem.commons.* import com.avsystem.commons.annotation.explicitGenerics +import com.avsystem.commons.serialization.json.{JsonReader, JsonStringInput} import io.udash.rest.raw.* +import io.udash.rest.util.Utils import io.udash.utils.URLEncoder import monix.eval.Task import monix.execution.Callback +import monix.reactive.Observable import org.eclipse.jetty.client.* import org.eclipse.jetty.http.{HttpCookie, HttpHeader, MimeTypes} @@ -14,75 +17,232 @@ import java.nio.charset.Charset import scala.concurrent.CancellationException import scala.concurrent.duration.* -object JettyRestClient { - final val DefaultMaxResponseLength = 2 * 1024 * 1024 - final val DefaultTimeout = 10.seconds +/** TODO streaming doc */ +final class JettyRestClient( + client: HttpClient, + defaultMaxResponseLength: Int = JettyRestClient.DefaultMaxResponseLength, + defaultTimeout: Duration = JettyRestClient.DefaultTimeout, +) { - @explicitGenerics def apply[RestApi: RawRest.AsRealRpc : RestMetadata]( - client: HttpClient, + @explicitGenerics + def create[RestApi: RawRest.AsRealRpc : RestMetadata]( baseUri: String, - maxResponseLength: Int = DefaultMaxResponseLength, - timeout: Duration = DefaultTimeout + customMaxResponseLength: OptArg[Int] = OptArg.Empty, + customTimeout: OptArg[Duration] = OptArg.Empty, ): RestApi = - RawRest.fromHandleRequest[RestApi](asHandleRequest(client, baseUri, maxResponseLength, timeout)) + RawRest.fromHandleRequestWithStreaming[RestApi]( + asHandleRequestWithStreaming(baseUri, customMaxResponseLength, customTimeout) + ) - def asHandleRequest( - client: HttpClient, + /** TODO streaming doc */ + def asHandleRequestWithStreaming( baseUrl: String, - maxResponseLength: Int = DefaultMaxResponseLength, - timeout: Duration = DefaultTimeout - ): RawRest.HandleRequest = - request => Task(client.newRequest(baseUrl).method(request.method.name)).flatMap { httpReq => - Task.async { (callback: Callback[Throwable, RestResponse]) => - val path = baseUrl + PlainValue.encodePath(request.parameters.path) + customMaxResponseLength: OptArg[Int] = OptArg.Empty, + customTimeout: OptArg[Duration] = OptArg.Empty, + ): RawRest.RestRequestHandler = new RawRest.RestRequestHandler { + private val timeout = customTimeout.getOrElse(defaultTimeout) + private val maxResponseLength = customMaxResponseLength.getOrElse(defaultMaxResponseLength) - httpReq.path(path) - request.parameters.query.entries.foreach { - case (name, PlainValue(value)) => httpReq.param(name, value) - } - request.parameters.headers.entries.foreach { - case (name, PlainValue(value)) => httpReq.headers(headers => headers.add(name, value)) - } - request.parameters.cookies.entries.foreach { - case (name, PlainValue(value)) => httpReq.cookie(HttpCookie.build( - URLEncoder.encode(name, spaceAsPlus = true), URLEncoder.encode(value, spaceAsPlus = true)).build()) - } + override def handleRequest(request: RestRequest): Task[RestResponse] = + prepareRequest(baseUrl, timeout, request).flatMap(sendRequest(_, maxResponseLength)) - request.body match { - case HttpBody.Empty => - case tb: HttpBody.Textual => - httpReq.body(new StringRequestContent(tb.contentType, tb.content, Charset.forName(tb.charset))) - case bb: HttpBody.Binary => - httpReq.body(new BytesRequestContent(bb.contentType, bb.bytes)) - } + override def handleRequestStream(request: RestRequest): Task[StreamedRestResponse] = + prepareRequest(baseUrl, timeout, request).flatMap { httpReq => + Task.async { (callback: Callback[Throwable, StreamedRestResponse]) => + val listener = new InputStreamResponseListener { + override def onHeaders(response: Response): Unit = { + super.onHeaders(response) + // TODO streaming document content length behaviour + val contentLength = response.getHeaders.getLongField(HttpHeader.CONTENT_LENGTH) + if (contentLength == -1) { + val contentTypeOpt = response.getHeaders.get(HttpHeader.CONTENT_TYPE).opt + val mediaTypeOpt = contentTypeOpt.map(MimeTypes.getContentTypeWithoutCharset) + val charsetOpt = contentTypeOpt.map(MimeTypes.getCharsetFromContentType) + // TODO streaming error handling client-side ??? + val bodyOpt = mediaTypeOpt matchOpt { + case Opt(HttpBody.OctetStreamType) => + // TODO streaming configure chunk size ??? + StreamedBody.RawBinary(Observable.fromInputStream(Task.eval(getInputStream))) + case Opt(HttpBody.JsonType) => + val charset = charsetOpt.getOrElse(HttpBody.Utf8Charset) + // suboptimal - maybe "online" parsing is possible using Jackson / other lib without waiting for full content ? + val elements: Observable[JsonValue] = + Observable + .fromTask(Utils.mergeArrays(Observable.fromInputStream(Task.eval(getInputStream)))) + .map(raw => new String(raw, charset)) + .flatMap { jsonStr => + val input = new JsonStringInput(new JsonReader(jsonStr)) + Observable + .fromIterator(Task.eval(input.readList().iterator(_.asInstanceOf[JsonStringInput].readRawJson()))) + .map(JsonValue(_)) + } + StreamedBody.JsonList( + elements = elements, + charset = charset, + ) + } + bodyOpt.mapOr( + { + // TODO streaming error handling client-side + callback(Failure(new Exception(s"Unsupported content type $contentTypeOpt"))) + }, + body => { + val restResponse = StreamedRestResponse( + code = response.getStatus, + headers = parseHeaders(response), + body = body, + batchSize = 1, + ) + callback(Success(restResponse)) + } + ) + } + } - timeout match { - case fd: FiniteDuration => httpReq.timeout(fd.length, fd.unit) - case _ => - } + override def onFailure(response: Response, failure: Throwable): Unit = { + super.onFailure(response, failure) + // TODO streaming error handling client-side ??? + } - httpReq.send(new BufferingResponseListener(maxResponseLength) { - override def onComplete(result: Result): Unit = - if (result.isSucceeded) { - val httpResp = result.getResponse + override def onComplete(result: Result): Unit = { + super.onComplete(result) + val httpResp = result.getResponse + val contentLength = httpResp.getHeaders.getLongField(HttpHeader.CONTENT_LENGTH) + if (contentLength != -1) { val contentTypeOpt = httpResp.getHeaders.get(HttpHeader.CONTENT_TYPE).opt val charsetOpt = contentTypeOpt.map(MimeTypes.getCharsetFromContentType) + // TODO streaming client-side handle errors ? + val rawBody = getInputStream.readAllBytes() val body = (contentTypeOpt, charsetOpt) match { case (Opt(contentType), Opt(charset)) => - HttpBody.textual(getContentAsString, MimeTypes.getContentTypeWithoutCharset(contentType), charset) + StreamedBody.fromHttpBody( + HttpBody.textual( + content = new String(rawBody, charset), + mediaType = MimeTypes.getContentTypeWithoutCharset(contentType), + charset = charset, + ) + ) case (Opt(contentType), Opt.Empty) => - HttpBody.binary(getContent, contentType) + StreamedBody.fromHttpBody(HttpBody.binary(rawBody, contentType)) case _ => - HttpBody.Empty + StreamedBody.Empty } - val headers = httpResp.getHeaders.asScala.iterator.map(h => (h.getName, PlainValue(h.getValue))).toList - val response = RestResponse(httpResp.getStatus, IMapping(headers), body) - callback(Success(response)) - } else { - callback(Failure(result.getFailure)) + val restResponse = StreamedRestResponse( + code = httpResp.getStatus, + headers = parseHeaders(httpResp), + body = body, + batchSize = 1, + ) + callback(Success(restResponse)) } - }) - } - .doOnCancel(Task(httpReq.abort(new CancellationException("Request cancelled")))) + } + } + httpReq.send(listener) + }.doOnCancel(Task(httpReq.abort(new CancellationException("Request cancelled")))) + } + } + + /** TODO streaming doc */ + def asHandleRequest( + baseUrl: String, + customMaxResponseLength: OptArg[Int] = OptArg.Empty, + customTimeout: OptArg[Duration] = OptArg.Empty, + ): RawRest.HandleRequest = { + val timeout = customTimeout.getOrElse(defaultTimeout) + val maxResponseLength = customMaxResponseLength.getOrElse(defaultMaxResponseLength) + request => prepareRequest(baseUrl, timeout, request).flatMap(sendRequest(_, maxResponseLength)) + } + + private def prepareRequest( + baseUrl: String, + timeout: Duration, + request: RestRequest, + ): Task[Request] = + Task(client.newRequest(baseUrl).method(request.method.name)).map { httpReq => + val path = baseUrl + PlainValue.encodePath(request.parameters.path) + + httpReq.path(path) + request.parameters.query.entries.foreach { + case (name, PlainValue(value)) => httpReq.param(name, value) + } + request.parameters.headers.entries.foreach { + case (name, PlainValue(value)) => httpReq.headers(headers => headers.add(name, value)) + } + request.parameters.cookies.entries.foreach { + case (name, PlainValue(value)) => httpReq.cookie(HttpCookie.build( + URLEncoder.encode(name, spaceAsPlus = true), URLEncoder.encode(value, spaceAsPlus = true)).build()) + } + + request.body match { + case HttpBody.Empty => + case tb: HttpBody.Textual => + httpReq.body(new StringRequestContent(tb.contentType, tb.content, Charset.forName(tb.charset))) + case bb: HttpBody.Binary => + httpReq.body(new BytesRequestContent(bb.contentType, bb.bytes)) + } + + timeout match { + case fd: FiniteDuration => httpReq.timeout(fd.length, fd.unit) + case _ => + } + httpReq } + + private def sendRequest(httpReq: Request, maxResponseLength: Int): Task[RestResponse] = + Task.async { (callback: Callback[Throwable, RestResponse]) => + httpReq.send(new BufferingResponseListener(maxResponseLength) { + override def onComplete(result: Result): Unit = + if (result.isSucceeded) { + val httpResp = result.getResponse + val contentTypeOpt = httpResp.getHeaders.get(HttpHeader.CONTENT_TYPE).opt + val charsetOpt = contentTypeOpt.map(MimeTypes.getCharsetFromContentType) + val body = (contentTypeOpt, charsetOpt) match { + case (Opt(contentType), Opt(charset)) => + HttpBody.textual(getContentAsString, MimeTypes.getContentTypeWithoutCharset(contentType), charset) + case (Opt(contentType), Opt.Empty) => + HttpBody.binary(getContent, contentType) + case _ => + HttpBody.Empty + } + val response = RestResponse(httpResp.getStatus, parseHeaders(httpResp), body) + callback(Success(response)) + } else { + callback(Failure(result.getFailure)) + } + }) + } + .doOnCancel(Task(httpReq.abort(new CancellationException("Request cancelled")))) + + private def parseHeaders(httpResp: Response): IMapping[PlainValue] = + IMapping(httpResp.getHeaders.asScala.iterator.map(h => (h.getName, PlainValue(h.getValue))).toList) +} + +object JettyRestClient { + final val DefaultMaxResponseLength = 2 * 1024 * 1024 + final val DefaultTimeout = 10.seconds + + @explicitGenerics + def apply[RestApi: RawRest.AsRealRpc : RestMetadata]( + client: HttpClient, + baseUri: String, + maxResponseLength: Int = DefaultMaxResponseLength, + timeout: Duration = DefaultTimeout, + ): RestApi = + new JettyRestClient(client, maxResponseLength, timeout).create[RestApi](baseUri) + + def asHandleRequest( + client: HttpClient, + baseUrl: String, + maxResponseLength: Int = DefaultMaxResponseLength, + timeout: Duration = DefaultTimeout, + ): RawRest.HandleRequest = + new JettyRestClient(client, maxResponseLength, timeout).asHandleRequest(baseUrl) + + def asHandleRequestWithStreaming( + client: HttpClient, + baseUrl: String, + maxResponseLength: Int = DefaultMaxResponseLength, + timeout: Duration = DefaultTimeout, + ): RawRest.RestRequestHandler = + new JettyRestClient(client, maxResponseLength, timeout).asHandleRequestWithStreaming(baseUrl) } diff --git a/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala b/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala index 22fb5ce69..b23793f72 100644 --- a/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala +++ b/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala @@ -1,11 +1,13 @@ package io.udash package rest.jetty -import io.udash.rest.raw.RawRest.HandleRequest -import io.udash.rest.{RestApiTestScenarios, ServletBasedRestApiTest} +import io.udash.rest.raw.RawRest +import io.udash.rest.{RestApiTestScenarios, ServletBasedRestApiTest, StreamingRestApiTestScenarios} import org.eclipse.jetty.client.HttpClient -final class JettyRestCallTest extends ServletBasedRestApiTest with RestApiTestScenarios { +final class JettyRestCallTest + extends ServletBasedRestApiTest with RestApiTestScenarios with StreamingRestApiTestScenarios { + /** * Similar to the default HttpClient, but with a connection timeout * significantly exceeding the value of the CallTimeout @@ -15,9 +17,12 @@ final class JettyRestCallTest extends ServletBasedRestApiTest with RestApiTestSc setIdleTimeout(IdleTimout.toMillis) } - def clientHandle: HandleRequest = + def clientHandle: RawRest.HandleRequest = JettyRestClient.asHandleRequest(client, s"$baseUrl/api", maxPayloadSize) + override def streamingClientHandler: RawRest.RestRequestHandler = + JettyRestClient.asHandleRequestWithStreaming(client, s"$baseUrl/stream-api", maxPayloadSize) + override protected def beforeAll(): Unit = { super.beforeAll() client.start() diff --git a/rest/src/main/scala/io/udash/rest/annotations.scala b/rest/src/main/scala/io/udash/rest/annotations.scala index 724e13106..f2ee0a389 100644 --- a/rest/src/main/scala/io/udash/rest/annotations.scala +++ b/rest/src/main/scala/io/udash/rest/annotations.scala @@ -320,6 +320,14 @@ trait ResponseAdjuster extends RealSymAnnotation { def adjustResponse(response: RestResponse): RestResponse } +/** + * Base trait for annotations which may be applied on REST API methods (including prefix methods) + * in order to customize outgoing response on the server side. + */ +trait StreamedResponseAdjuster extends RealSymAnnotation { + def adjustResponse(response: StreamedRestResponse): StreamedRestResponse +} + /** * Convenience implementation of [[RequestAdjuster]]. */ @@ -334,6 +342,13 @@ class adjustResponse(f: RestResponse => RestResponse) extends ResponseAdjuster { def adjustResponse(response: RestResponse): RestResponse = f(response) } +/** + * Convenience implementation of [[StreamedResponseAdjuster]]. + */ +class adjustStreamedResponse(f: StreamedRestResponse => StreamedRestResponse) extends StreamedResponseAdjuster { + def adjustResponse(response: StreamedRestResponse): StreamedRestResponse = f(response) +} + /** * Annotation which may be applied on REST API methods (including prefix methods) in order to append additional * HTTP header to all outgoing requests generated for invocations of that method on the client side. @@ -346,6 +361,7 @@ class addRequestHeader(name: String, value: String) extends RequestAdjuster { * Annotation which may be applied on REST API methods (including prefix methods) in order to append additional * HTTP header to all outgoing responses generated for invocations of that method on the server side. */ -class addResponseHeader(name: String, value: String) extends ResponseAdjuster { +class addResponseHeader(name: String, value: String) extends ResponseAdjuster with StreamedResponseAdjuster { def adjustResponse(response: RestResponse): RestResponse = response.header(name, value) + override def adjustResponse(response: StreamedRestResponse): StreamedRestResponse = response.header(name, value) } diff --git a/rest/src/main/scala/io/udash/rest/openapi/OpenApiMetadata.scala b/rest/src/main/scala/io/udash/rest/openapi/OpenApiMetadata.scala index fc97821bc..d095d4893 100644 --- a/rest/src/main/scala/io/udash/rest/openapi/OpenApiMetadata.scala +++ b/rest/src/main/scala/io/udash/rest/openapi/OpenApiMetadata.scala @@ -48,12 +48,12 @@ final case class OpenApiMetadata[T]( @tagged[SomeBodyTag](whenUntagged = new JsonBody) @paramTag[RestParamTag](defaultTag = new Body) @unmatched(RawRest.NotValidHttpMethod) - bodyMethods: List[OpenApiBodyOperation[_]] + bodyMethods: List[OpenApiBodyOperation[_]], ) { val httpMethods: List[OpenApiOperation[_]] = (gets: List[OpenApiOperation[_]]) ++ customBodyMethods ++ bodyMethods // collect all tags - private lazy val openApiTags: List[Tag] = { + lazy val openApiTags: List[Tag] = { def createTag(method: OpenApiMethod[_]): Opt[Tag] = method.groupAnnot.map { group => method.tagAdjusters.foldLeft(Tag(group.groupName))({ case (tag, adjuster) => adjuster.adjustTag(tag) }) @@ -102,7 +102,7 @@ final case class OpenApiMetadata[T]( servers: List[Server] = Nil, security: List[SecurityRequirement] = Nil, tags: List[Tag] = Nil, - externalDocs: OptArg[ExternalDocumentation] = OptArg.Empty + externalDocs: OptArg[ExternalDocumentation] = OptArg.Empty, ): OpenApi = { val registry = new SchemaRegistry(initial = components.schemas) OpenApi( diff --git a/rest/src/main/scala/io/udash/rest/openapi/RestSchema.scala b/rest/src/main/scala/io/udash/rest/openapi/RestSchema.scala index 8cb5d1391..dc6cffad9 100644 --- a/rest/src/main/scala/io/udash/rest/openapi/RestSchema.scala +++ b/rest/src/main/scala/io/udash/rest/openapi/RestSchema.scala @@ -2,10 +2,11 @@ package io.udash package rest.openapi import java.util.UUID -import com.avsystem.commons._ +import com.avsystem.commons.* import com.avsystem.commons.misc.{ImplicitNotFound, NamedEnum, NamedEnumCompanion, Timestamp} -import io.udash.rest.raw._ +import io.udash.rest.raw.* import monix.eval.TaskLike +import monix.reactive.Observable import scala.annotation.implicitNotFound @@ -180,10 +181,20 @@ object RestMediaTypes { Map(HttpBody.OctetStreamType -> MediaType(schema = schema)) } + implicit val ByteArrayStreamMediaTypes: RestMediaTypes[Observable[Array[Byte]]] = + (resolver: SchemaResolver, schemaTransform: RestSchema[_] => RestSchema[_]) => { + val schema = resolver.resolve(schemaTransform(RestSchema.plain(Schema.Binary))) + Map(HttpBody.OctetStreamType -> MediaType(schema = schema)) + } + implicit def fromSchema[T: RestSchema]: RestMediaTypes[T] = (resolver: SchemaResolver, schemaTransform: RestSchema[_] => RestSchema[_]) => Map(HttpBody.JsonType -> MediaType(schema = resolver.resolve(schemaTransform(RestSchema[T])))) + implicit def jsonStreamMediaTypes[T: RestSchema]: RestMediaTypes[Observable[T]] = + (resolver: SchemaResolver, schemaTransform: RestSchema[_] => RestSchema[_]) => + Map(HttpBody.JsonType -> MediaType(schema = RefOr(Schema.arrayOf(resolver.resolve(schemaTransform(RestSchema[T])))))) + @implicitNotFound("RestMediaTypes instance for ${T} not found, because:\n#{forSchema}") implicit def notFound[T](implicit forSchema: ImplicitNotFound[RestSchema[T]]): ImplicitNotFound[RestMediaTypes[T]] = ImplicitNotFound() @@ -251,6 +262,9 @@ object RestResultType { implicit def forAsyncEffect[F[_] : TaskLike, T: RestResponses]: RestResultType[F[T]] = RestResultType(RestResponses[T].responses(_, identity)) + implicit def forObservable[T](implicit rr: RestResponses[Observable[T]]): RestResultType[Observable[T]] = + RestResultType(rr.responses(_, identity)) + @implicitNotFound("#{forResponseType}") implicit def notFound[T]( implicit forResponseType: ImplicitNotFound[HttpResponseType[T]] diff --git a/rest/src/main/scala/io/udash/rest/raw/HttpBody.scala b/rest/src/main/scala/io/udash/rest/raw/HttpBody.scala index 3da4b2c10..78ac1835c 100644 --- a/rest/src/main/scala/io/udash/rest/raw/HttpBody.scala +++ b/rest/src/main/scala/io/udash/rest/raw/HttpBody.scala @@ -33,7 +33,10 @@ sealed trait HttpBody { final def readForm(defaultCharset: String = HttpBody.Utf8Charset): String = readText(HttpBody.FormType, defaultCharset) - final def readText(requiredMediaType: OptArg[String] = OptArg.Empty, defaultCharset: String = HttpBody.Utf8Charset): String = this match { + final def readText( + requiredMediaType: OptArg[String] = OptArg.Empty, + defaultCharset: String = HttpBody.Utf8Charset, + ): String = this match { case HttpBody.Empty => throw new ReadFailure("Expected non-empty textual body") case ne: HttpBody.NonEmpty if requiredMediaType.forall(_ == ne.mediaType) => diff --git a/rest/src/main/scala/io/udash/rest/raw/RawRest.scala b/rest/src/main/scala/io/udash/rest/raw/RawRest.scala index 6bcf8fe48..61d247b3b 100644 --- a/rest/src/main/scala/io/udash/rest/raw/RawRest.scala +++ b/rest/src/main/scala/io/udash/rest/raw/RawRest.scala @@ -32,13 +32,16 @@ final case class ResolvedCall(root: RestMetadata[_], prefixes: List[PrefixCall], def adjustResponse(response: Task[RestResponse]): Task[RestResponse] = prefixes.foldRight(finalCall.metadata.adjustResponse(response))(_.metadata.adjustResponse(_)) + + def adjustResponseWithStreaming(response: Task[AbstractRestResponse]): Task[AbstractRestResponse] = + prefixes.foldRight(finalCall.metadata.adjustResponseWithStreaming(response))(_.metadata.adjustResponseWithStreaming(_)) } @methodTag[RestMethodTag] @methodTag[BodyTypeTag] trait RawRest { - import RawRest._ + import RawRest.* // declaration order of raw methods matters - it determines their priority! @@ -50,7 +53,7 @@ trait RawRest { @unmatchedParam[Body](RawRest.PrefixMethodBodyParam) def prefix( @methodName name: String, - @composite parameters: RestParameters + @composite parameters: RestParameters, ): Try[RawRest] @multi @tried @@ -61,9 +64,20 @@ trait RawRest { @unmatchedParam[Body](RawRest.GetMethodBodyParam) def get( @methodName name: String, - @composite parameters: RestParameters + @composite parameters: RestParameters, ): Task[RestResponse] + @multi @tried + @tagged[GET] + @tagged[NoBody](whenUntagged = new NoBody) + @paramTag[RestParamTag](defaultTag = new Query) + @unmatched(RawRest.NotValidGetStreamMethod) + @unmatchedParam[Body](RawRest.GetMethodBodyParam) + def getStream( + @methodName name: String, + @composite parameters: RestParameters, + ): Task[StreamedRestResponse] + @multi @tried @tagged[BodyMethodTag](whenUntagged = new POST) @tagged[FormBody] @@ -72,7 +86,7 @@ trait RawRest { def handleForm( @methodName name: String, @composite parameters: RestParameters, - @multi @tagged[Body] body: Mapping[PlainValue] + @multi @tagged[Body] body: Mapping[PlainValue], ): Task[RestResponse] @multi @tried @@ -83,9 +97,20 @@ trait RawRest { def handleJson( @methodName name: String, @composite parameters: RestParameters, - @multi @tagged[Body] body: Mapping[JsonValue] + @multi @tagged[Body] body: Mapping[JsonValue], ): Task[RestResponse] + @multi @tried + @tagged[BodyMethodTag](whenUntagged = new POST) + @tagged[JsonBody](whenUntagged = new JsonBody) + @paramTag[RestParamTag](defaultTag = new Body) + @unmatched(RawRest.NotValidHttpMethodStream) + def handleJsonStream( + @methodName name: String, + @composite parameters: RestParameters, + @multi @tagged[Body] body: Mapping[JsonValue], + ): Task[StreamedRestResponse] + @multi @tried @tagged[BodyMethodTag](whenUntagged = new POST) @tagged[CustomBody] @@ -95,13 +120,32 @@ trait RawRest { def handleCustom( @methodName name: String, @composite parameters: RestParameters, - @encoded @tagged[Body] @unmatched(RawRest.MissingBodyParam) body: HttpBody + @encoded @tagged[Body] @unmatched(RawRest.MissingBodyParam) body: HttpBody, ): Task[RestResponse] + @multi @tried + @tagged[BodyMethodTag](whenUntagged = new POST) + @tagged[CustomBody] + @paramTag[RestParamTag](defaultTag = new Body) + @unmatched(RawRest.NotValidCustomBodyStreamMethod) + @unmatchedParam[Body](RawRest.SuperfluousBodyParam) + def handleCustomStream( + @methodName name: String, + @composite parameters: RestParameters, + @encoded @tagged[Body] @unmatched(RawRest.MissingBodyParam) body: HttpBody, + ): Task[StreamedRestResponse] + def asHandleRequest(metadata: RestMetadata[_]): HandleRequest = - RawRest.resolveAndHandle(metadata)(handleResolved) + RawRest.resolveAndHandle(metadata)(handleResolved).andThen(StreamedRestResponse.fallbackToRestResponse) + + def asHandleRequestWithStreaming(metadata: RestMetadata[_]): HandleRequestWithStreaming = + RawRest.resolveAndHandle(metadata)(handleResolvedWithStreaming) + + // TODO doc for compatibility + def handleResolved(request: RestRequest, resolved: ResolvedCall): Task[RestResponse] = + StreamedRestResponse.fallbackToRestResponse(handleResolvedWithStreaming(request, resolved)) - def handleResolved(request: RestRequest, resolved: ResolvedCall): Task[RestResponse] = { + def handleResolvedWithStreaming(request: RestRequest, resolved: ResolvedCall): Task[AbstractRestResponse] = { val RestRequest(method, parameters, body) = request val ResolvedCall(_, prefixes, finalCall) = resolved val HttpCall(finalPathParams, finalMetadata) = finalCall @@ -111,7 +155,7 @@ trait RawRest { } @tailrec - def resolveCall(rawRest: RawRest, prefixes: List[PrefixCall]): Task[RestResponse] = prefixes match { + def resolveCall(rawRest: RawRest, prefixes: List[PrefixCall]): Task[AbstractRestResponse] = prefixes match { case PrefixCall(pathParams, pm) :: tail => rawRest.prefix(pm.name, parameters.copy(path = pathParams)) match { case Success(nextRawRest) => resolveCall(nextRawRest, tail) @@ -120,16 +164,22 @@ trait RawRest { } case Nil => val finalParameters = parameters.copy(path = finalPathParams) - if (method == HttpMethod.GET) - rawRest.get(finalMetadata.name, finalParameters) - else if (finalMetadata.customBody) - rawRest.handleCustom(finalMetadata.name, finalParameters, body) - else if (finalMetadata.formBody) + if (method == HttpMethod.GET) { + if (!finalMetadata.streamedResponse) rawRest.get(finalMetadata.name, finalParameters) + else rawRest.getStream(finalMetadata.name, finalParameters) + } else if (finalMetadata.customBody) { + if (!finalMetadata.streamedResponse) rawRest.handleCustom(finalMetadata.name, finalParameters, body) + else rawRest.handleCustomStream(finalMetadata.name, finalParameters, body) + } else if (finalMetadata.formBody) { rawRest.handleForm(finalMetadata.name, finalParameters, handleBadBody(HttpBody.parseFormBody(body))) - else - rawRest.handleJson(finalMetadata.name, finalParameters, handleBadBody(HttpBody.parseJsonBody(body))) + } else { + if (!finalMetadata.streamedResponse) + rawRest.handleJson(finalMetadata.name, finalParameters, handleBadBody(HttpBody.parseJsonBody(body))) + else + rawRest.handleJsonStream(finalMetadata.name, finalParameters, handleBadBody(HttpBody.parseJsonBody(body))) + } } - try resolved.adjustResponse(resolveCall(this, prefixes)) catch { + try resolved.adjustResponseWithStreaming(resolveCall(this, prefixes)) catch { case e: InvalidRpcCall => Task.now(extractHttpException(e).map(_.toResponse).getOrElse(RestResponse.plain(400, e.getMessage))) } @@ -144,11 +194,18 @@ trait RawRest { object RawRest extends RawRpcCompanion[RawRest] { type HandleRequest = RestRequest => Task[RestResponse] + type HandleRequestWithStreaming = RestRequest => Task[AbstractRestResponse] + + trait RestRequestHandler { + def handleRequest(request: RestRequest): Task[RestResponse] + def handleRequestStream(request: RestRequest): Task[StreamedRestResponse] + } /** * Similar to [[io.udash.rest.raw.RawRest.HandleRequest HandleRequest]] but accepts already resolved path as a second argument. */ type HandleResolvedRequest = (RestRequest, ResolvedCall) => Task[RestResponse] + type HandleResolvedRequestWithStreaming = (RestRequest, ResolvedCall) => Task[AbstractRestResponse] type AsTask[F[_]] = TaskLike[F] trait FromTask[F[_]] { @@ -166,14 +223,20 @@ object RawRest extends RawRpcCompanion[RawRest] { "prefix methods cannot take @Body parameters" final val NotValidGetMethod = "it cannot be translated into an HTTP GET method" + final val NotValidGetStreamMethod = + "it cannot be translated into an HTTP GET stream method" final val GetMethodBodyParam = "GET methods cannot take @Body parameters" final val NotValidHttpMethod = "it cannot be translated into an HTTP method" + final val NotValidHttpMethodStream = + "it cannot be translated into an HTTP stream method" final val NotValidFormBodyMethod = "it cannot be translated into an HTTP method with form body" final val NotValidCustomBodyMethod = "it cannot be translated into an HTTP method with custom body" + final val NotValidCustomBodyStreamMethod = + "it cannot be translated into an HTTP stream method with custom body" final val MissingBodyParam = "expected exactly one @Body parameter but none was found" final val SuperfluousBodyParam = @@ -187,13 +250,27 @@ object RawRest extends RawRpcCompanion[RawRest] { @implicitNotFound(InvalidTraitMessage) implicit def rawRestAsRawNotFound[T]: ImplicitNotFound[AsRaw[RawRest, T]] = ImplicitNotFound() - def fromHandleRequest[Real: AsRealRpc : RestMetadata](handleRequest: HandleRequest): Real = + // client side + def fromHandleRequest[Real: AsRealRpc : RestMetadata](handle: HandleRequest): Real = + RawRest.asReal(new DefaultRawRest(Nil, RestMetadata[Real], RestParameters.Empty, new RawRest.RestRequestHandler { + override def handleRequest(request: RestRequest): Task[RestResponse] = handle(request) + override def handleRequestStream(request: RestRequest): Task[StreamedRestResponse] = + Task.raiseError(new UnsupportedOperationException("Streaming unsupported by the client")) + })) + + // client side with response streaming support + def fromHandleRequestWithStreaming[Real: AsRealRpc : RestMetadata](handleRequest: RawRest.RestRequestHandler): Real = RawRest.asReal(new DefaultRawRest(Nil, RestMetadata[Real], RestParameters.Empty, handleRequest)) + // server side def asHandleRequest[Real: AsRawRpc : RestMetadata](real: Real): HandleRequest = RawRest.asRaw(real).asHandleRequest(RestMetadata[Real]) - def resolveAndHandle(metadata: RestMetadata[_])(handleResolved: HandleResolvedRequest): HandleRequest = { + // server side with response streaming support + def asHandleRequestWithStreaming[Real: AsRawRpc : RestMetadata](real: Real): HandleRequestWithStreaming = + RawRest.asRaw(real).asHandleRequestWithStreaming(RestMetadata[Real]) + + def resolveAndHandle(metadata: RestMetadata[_])(handleResolved: HandleResolvedRequestWithStreaming): HandleRequestWithStreaming = { metadata.ensureValid() request => { @@ -208,16 +285,20 @@ object RawRest extends RawRpcCompanion[RawRest] { case HttpMethod.GET => List(HttpMethod.GET, HttpMethod.HEAD) case m => List(m) } ++ Iterator(HttpMethod.OPTIONS) - val response = RestResponse(200, - IMapping.create("Allow" -> PlainValue(meths.mkString(","))), HttpBody.Empty) - Task.now(response) + Task.now(RestResponse(200, IMapping.create("Allow" -> PlainValue(meths.mkString(","))), HttpBody.Empty)) case wireMethod => val head = wireMethod == HttpMethod.HEAD val req = if (head) request.copy(method = HttpMethod.GET) else request calls.find(_.method == req.method) match { case Some(call) => val resp = handleResolved(req, call) - if (head) resp.map(_.copy(body = HttpBody.empty)) else resp + if (head) + resp.map { + case resp: RestResponse => resp.copy(body = HttpBody.empty) + case stream: StreamedRestResponse => stream.copy(body = StreamedBody.empty) + } + else + resp case None => val message = s"$wireMethod not allowed on path ${PlainValue.encodePath(path)}" Task.now(RestResponse.plain(405, message)) @@ -231,7 +312,7 @@ object RawRest extends RawRpcCompanion[RawRest] { prefixMetas: List[PrefixMetadata[_]], //in reverse invocation order! metadata: RestMetadata[_], prefixParams: RestParameters, - handleRequest: HandleRequest + handleRequest: RawRest.RestRequestHandler, ) extends RawRest { def prefix(name: String, parameters: RestParameters): Try[RawRest] = @@ -243,21 +324,52 @@ object RawRest extends RawRpcCompanion[RawRest] { def get(name: String, parameters: RestParameters): Task[RestResponse] = doHandle("get", name, parameters, HttpBody.Empty) + def getStream(name: String, parameters: RestParameters): Task[StreamedRestResponse] = + doHandleStream("getStream", name, parameters, HttpBody.Empty) + def handleJson(name: String, parameters: RestParameters, body: Mapping[JsonValue]): Task[RestResponse] = doHandle("handle", name, parameters, HttpBody.createJsonBody(body)) + def handleJsonStream(name: String, parameters: RestParameters, body: Mapping[JsonValue]): Task[StreamedRestResponse] = + doHandleStream("handleStream", name, parameters, HttpBody.Empty) + def handleForm(name: String, parameters: RestParameters, body: Mapping[PlainValue]): Task[RestResponse] = doHandle("handleForm", name, parameters, HttpBody.createFormBody(body)) def handleCustom(name: String, parameters: RestParameters, body: HttpBody): Task[RestResponse] = doHandle("handleSingle", name, parameters, body) + def handleCustomStream(name: String, parameters: RestParameters, body: HttpBody): Task[StreamedRestResponse] = + doHandleStream("handleSingleStream", name, parameters, body) + private def doHandle(rawName: String, name: String, parameters: RestParameters, body: HttpBody): Task[RestResponse] = - metadata.httpMethodsByName.get(name).map { methodMeta => - val newHeaders = prefixParams.append(methodMeta, parameters) - val baseRequest = RestRequest(methodMeta.method, newHeaders, body) - val request = prefixMetas.foldLeft(methodMeta.adjustRequest(baseRequest))((req, meta) => meta.adjustRequest(req)) - handleRequest(request) - } getOrElse Task.raiseError(new UnknownRpc(name, rawName)) + metadata.httpMethodsByName.getOpt(name) + .collect { case methodMeta if !methodMeta.streamedResponse => + handleRequest.handleRequest(resolveRequest(parameters, body, methodMeta)) + } + .getOrElse(Task.raiseError(new UnknownRpc(name, rawName))) + + private def doHandleStream( + rawName: String, + name: String, + parameters: RestParameters, + body: HttpBody, + ): Task[StreamedRestResponse] = + metadata.httpMethodsByName.getOpt(name) + .collect { case methodMeta if methodMeta.streamedResponse => + handleRequest.handleRequestStream(resolveRequest(parameters, body, methodMeta)) + } + .getOrElse(Task.raiseError(new UnknownRpc(name, rawName))) + + private def resolveRequest( + parameters: RestParameters, + body: HttpBody, + methodMeta: HttpMethodMetadata[_], + ): RestRequest = { + val newHeaders = prefixParams.append(methodMeta, parameters) + val baseRequest = RestRequest(methodMeta.method, newHeaders, body) + val request = prefixMetas.foldLeft(methodMeta.adjustRequest(baseRequest))((req, meta) => meta.adjustRequest(req)) + request + } } } diff --git a/rest/src/main/scala/io/udash/rest/raw/RestMetadata.scala b/rest/src/main/scala/io/udash/rest/raw/RestMetadata.scala index 5151ad739..869e910fd 100644 --- a/rest/src/main/scala/io/udash/rest/raw/RestMetadata.scala +++ b/rest/src/main/scala/io/udash/rest/raw/RestMetadata.scala @@ -2,12 +2,13 @@ package io.udash package rest package raw -import com.avsystem.commons._ -import com.avsystem.commons.meta._ -import com.avsystem.commons.rpc._ +import com.avsystem.commons.* +import com.avsystem.commons.meta.* +import com.avsystem.commons.rpc.* import io.udash.macros.RestMacros import io.udash.rest.raw.RestMetadata.ResolutionTrie import monix.eval.{Task, TaskLike} +import monix.reactive.Observable import scala.annotation.implicitNotFound @@ -37,7 +38,7 @@ final case class RestMetadata[T]( @tagged[SomeBodyTag](whenUntagged = new JsonBody) @paramTag[RestParamTag](defaultTag = new Body) @unmatched(RawRest.NotValidHttpMethod) - @rpcMethodMetadata httpBodyMethods: List[HttpMethodMetadata[_]] + @rpcMethodMetadata httpBodyMethods: List[HttpMethodMetadata[_]], ) { val httpMethods: List[HttpMethodMetadata[_]] = httpGetMethods ++ httpBodyMethods @@ -234,6 +235,7 @@ sealed abstract class RestMethodMetadata[T] extends TypedMetadata[T] { def parametersMetadata: RestParametersMetadata def requestAdjusters: List[RequestAdjuster] def responseAdjusters: List[ResponseAdjuster] + def streamedResponseAdjusters: List[StreamedResponseAdjuster] val pathPattern: List[PathPatternElement] = methodPath.map(PathName.apply) ++ parametersMetadata.pathParams.flatMap(pp => PathParam(pp) :: pp.pathSuffix.map(PathName.apply)) @@ -269,6 +271,16 @@ sealed abstract class RestMethodMetadata[T] extends TypedMetadata[T] { def adjustResponse(asyncResponse: Task[RestResponse]): Task[RestResponse] = if (responseAdjusters.isEmpty) asyncResponse else asyncResponse.map(resp => responseAdjusters.foldRight(resp)(_ adjustResponse _)) + + def adjustResponseWithStreaming(asyncResponse: Task[AbstractRestResponse]): Task[AbstractRestResponse] = + asyncResponse.map { + case resp: RestResponse => + if (responseAdjusters.isEmpty) resp + else responseAdjusters.foldRight(resp)(_ adjustResponse _) + case stream: StreamedRestResponse => + if (streamedResponseAdjusters.isEmpty) stream + else streamedResponseAdjusters.foldRight(stream)(_ adjustResponse _) + } } final case class PrefixMetadata[T]( @@ -277,7 +289,8 @@ final case class PrefixMetadata[T]( @composite parametersMetadata: RestParametersMetadata, @multi @reifyAnnot requestAdjusters: List[RequestAdjuster], @multi @reifyAnnot responseAdjusters: List[ResponseAdjuster], - @infer @checked result: RestMetadata.Lazy[T] + @multi @reifyAnnot streamedResponseAdjusters: List[StreamedResponseAdjuster], + @infer @checked result: RestMetadata.Lazy[T], ) extends RestMethodMetadata[T] { def methodPath: List[PlainValue] = PlainValue.decodePath(methodTag.path) } @@ -291,7 +304,8 @@ final case class HttpMethodMetadata[T]( @isAnnotated[FormBody] formBody: Boolean, @multi @reifyAnnot requestAdjusters: List[RequestAdjuster], @multi @reifyAnnot responseAdjusters: List[ResponseAdjuster], - @infer @checked responseType: HttpResponseType[T] + @multi @reifyAnnot streamedResponseAdjusters: List[StreamedResponseAdjuster], + @infer @checked responseType: HttpResponseType[T], ) extends RestMethodMetadata[T] { val method: HttpMethod = methodTag.method @@ -300,6 +314,8 @@ final case class HttpMethodMetadata[T]( case _ => false } + val streamedResponse: Boolean = responseType.streamed + def singleBodyParam: Opt[ParamMetadata[_]] = if (customBody) bodyParams.headOpt else Opt.Empty @@ -321,8 +337,16 @@ final case class HttpMethodMetadata[T]( * See `MacroInstances` for more information on injection of implicits. */ @implicitNotFound("${T} is not a valid result type of HTTP REST method") -final case class HttpResponseType[T]() -object HttpResponseType { +final case class HttpResponseType[T](streamed: Boolean = false) +object HttpResponseType extends HttpResponseTypeLowPrio { + + implicit def observableResponseType[T]: HttpResponseType[Observable[T]] = + HttpResponseType(streamed = true) + + implicit def streamedResponseType[F[_] : TaskLike, T](implicit asRaw: AsRaw[StreamedRestResponse, T]): HttpResponseType[F[T]] = + HttpResponseType(streamed = true) +} +trait HttpResponseTypeLowPrio { this: HttpResponseType.type => implicit def asyncEffectResponseType[F[_] : TaskLike, T]: HttpResponseType[F[T]] = HttpResponseType() } @@ -347,7 +371,7 @@ final case class ParamMetadata[T]( final case class PathParamMetadata[T]( @reifyName(useRawName = true) name: String, - @reifyAnnot pathAnnot: Path + @reifyAnnot pathAnnot: Path, ) extends TypedMetadata[T] { val pathSuffix: List[PlainValue] = PlainValue.decodePath(pathAnnot.pathSuffix) } diff --git a/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala b/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala index e9a98752f..806d12813 100644 --- a/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala +++ b/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala @@ -5,18 +5,34 @@ import com.avsystem.commons._ import com.avsystem.commons.misc.ImplicitNotFound import com.avsystem.commons.rpc.{AsRaw, AsReal} import io.udash.rest.raw.RawRest.FromTask +import io.udash.rest.raw.StreamedBody.castOrFail +import io.udash.rest.util.Utils import monix.eval.{Task, TaskLike} +import monix.reactive.Observable import scala.annotation.implicitNotFound -final case class RestResponse(code: Int, headers: IMapping[PlainValue], body: HttpBody) { +/** TODO streaming doc */ +sealed trait AbstractRestResponse { + def code: Int + def headers: IMapping[PlainValue] + + final def isSuccess: Boolean = code >= 200 && code < 300 +} + +/** TODO streaming doc */ +final case class RestResponse( + code: Int, + headers: IMapping[PlainValue], + body: HttpBody, +) extends AbstractRestResponse { + def header(name: String, value: String): RestResponse = copy(headers = headers.append(name, PlainValue(value))) - def isSuccess: Boolean = - code >= 200 && code < 300 def toHttpError: HttpErrorException = HttpErrorException(code, body) + def ensureNonError: RestResponse = if (isSuccess) this else throw toHttpError } @@ -95,3 +111,134 @@ trait RestResponseLowPrio { this: RestResponse.type => implicit forBody: ImplicitNotFound[AsRaw[HttpBody, T]] ): ImplicitNotFound[AsRaw[RestResponse, T]] = ImplicitNotFound() } + +/** TODO streaming doc */ +final case class StreamedRestResponse( + code: Int, + headers: IMapping[PlainValue], + body: StreamedBody, + batchSize: Int, +) extends AbstractRestResponse { + + def header(name: String, value: String): StreamedRestResponse = + copy(headers = headers.append(name, PlainValue(value))) + + def ensureNonError: StreamedRestResponse = + if (isSuccess) this else throw HttpErrorException(code, StreamedBody.toHttpBody(body)) +} + +object StreamedRestResponse extends StreamedRestResponseLowPrio { + + /** TODO streaming doc */ + def fallbackToRestResponse(response: StreamedRestResponse): Task[RestResponse] = { + val httpBody: Task[HttpBody] = response.body match { + case StreamedBody.Empty => + Task.now(HttpBody.Empty) + case binary: StreamedBody.RawBinary => + Utils.mergeArrays(binary.content).map(HttpBody.Binary(_, binary.contentType)) + case jsonList: StreamedBody.JsonList => + jsonList.elements + .foldLeftL(new StringBuilder("[")) { case (sb, json) => + if (sb.sizeCompare(1) > 0) { + sb.append(',') + } + sb.append(json.value) + } + .map(_.append(']').result()) + .map(rawJson => HttpBody.json(JsonValue(rawJson))) + case single: StreamedBody.Single => + Task.now(single.body) + } + httpBody.map(RestResponse(response.code, response.headers, _)) + } + + /** TODO doc */ + def fallbackToRestResponse(response: Task[AbstractRestResponse]): Task[RestResponse] = + response.flatMap { + case restResponse: RestResponse => Task.now(restResponse) + case streamedResponse: StreamedRestResponse => fallbackToRestResponse(streamedResponse) + } + + def fromHttpError(error: HttpErrorException): StreamedRestResponse = + StreamedRestResponse(error.code, IMapping.empty, StreamedBody.fromHttpBody(error.payload), 1) + + class LazyOps(private val resp: () => StreamedRestResponse) extends AnyVal { + def recoverHttpError: StreamedRestResponse = try resp() catch { + case e: HttpErrorException => StreamedRestResponse.fromHttpError(e) + } + } + implicit def lazyOps(resp: => StreamedRestResponse): LazyOps = new LazyOps(() => resp) + + implicit class TaskOps(private val asyncResp: Task[StreamedRestResponse]) extends AnyVal { + def recoverHttpError: Task[StreamedRestResponse] = + asyncResp.onErrorRecover { + case e: HttpErrorException => StreamedRestResponse.fromHttpError(e) + } + } + + implicit def taskLikeFromResponseTask[F[_], T]( + implicit fromTask: FromTask[F], fromResponse: AsReal[StreamedRestResponse, T] + ): AsReal[Task[StreamedRestResponse], Try[F[T]]] = + rawTask => Success(fromTask.fromTask(rawTask.map(fromResponse.asReal))) + + implicit def taskLikeToResponseTask[F[_], T]( + implicit taskLike: TaskLike[F], asResponse: AsRaw[StreamedRestResponse, T] + ): AsRaw[Task[StreamedRestResponse], Try[F[T]]] = + _.fold(Task.raiseError, ft => Task.from(ft).map(asResponse.asRaw)).recoverHttpError + + implicit def observableFromResponseTask[T]( + implicit fromResponse: AsReal[StreamedRestResponse, Observable[T]] + ): AsReal[Task[StreamedRestResponse], Try[Observable[T]]] = + rawTask => Success(Observable.fromTask(rawTask).flatMap(fromResponse.asReal)) + + implicit def observableToResponseTask[T]( + implicit asResponse: AsRaw[StreamedRestResponse, Observable[T]] + ): AsRaw[Task[StreamedRestResponse], Try[Observable[T]]] = + _.fold(Task.raiseError, ft => Task.eval(ft).map(asResponse.asRaw)).recoverHttpError + + // following two implicits provide nice error messages when serialization is lacking for HTTP method result + // while the async wrapper is fine (e.g. Future) + + @implicitNotFound("${F}[${T}] is not a valid result type because:\n#{forResponseType}") + implicit def effAsyncAsRealNotFound[F[_], T](implicit + fromAsync: TaskLike[F], + forResponseType: ImplicitNotFound[AsReal[StreamedRestResponse, T]] + ): ImplicitNotFound[AsReal[Task[StreamedRestResponse], Try[F[T]]]] = ImplicitNotFound() + + @implicitNotFound("${F}[${T}] is not a valid result type because:\n#{forResponseType}") + implicit def effAsyncAsRawNotFound[F[_], T](implicit + toAsync: TaskLike[F], + forResponseType: ImplicitNotFound[AsRaw[StreamedRestResponse, T]] + ): ImplicitNotFound[AsRaw[Task[StreamedRestResponse], Try[F[T]]]] = ImplicitNotFound() + + // following two implicits provide nice error messages when result type of HTTP method is totally wrong + + @implicitNotFound("#{forResponseType}") + implicit def asyncAsRealNotFound[T]( + implicit forResponseType: ImplicitNotFound[HttpResponseType[T]] + ): ImplicitNotFound[AsReal[Task[StreamedRestResponse], Try[T]]] = ImplicitNotFound() + + @implicitNotFound("#{forResponseType}") + implicit def asyncAsRawNotFound[T]( + implicit forResponseType: ImplicitNotFound[HttpResponseType[T]] + ): ImplicitNotFound[AsRaw[Task[StreamedRestResponse], Try[T]]] = ImplicitNotFound() +} +trait StreamedRestResponseLowPrio { this: StreamedRestResponse.type => + implicit def bodyBasedFromResponse[T](implicit bodyAsReal: AsReal[StreamedBody, T]): AsReal[StreamedRestResponse, T] = + resp => bodyAsReal.asReal(resp.ensureNonError.body) + + implicit def bodyBasedToResponse[T](implicit bodyAsRaw: AsRaw[StreamedBody, T]): AsRaw[StreamedRestResponse, T] = + value => bodyAsRaw.asRaw(value).defaultResponse.recoverHttpError + + // following two implicits forward implicit-not-found error messages for StreamedBody as error messages for StreamedRestResponse + + @implicitNotFound("Cannot deserialize ${T} from StreamedRestResponse, because:\n#{forBody}") + implicit def asRealNotFound[T]( + implicit forBody: ImplicitNotFound[AsReal[StreamedBody, T]] + ): ImplicitNotFound[AsReal[StreamedRestResponse, T]] = ImplicitNotFound() + + @implicitNotFound("Cannot serialize ${T} into StreamedRestResponse, because:\n#{forBody}") + implicit def asRawNotFound[T]( + implicit forBody: ImplicitNotFound[AsRaw[StreamedBody, T]] + ): ImplicitNotFound[AsRaw[StreamedRestResponse, T]] = ImplicitNotFound() +} diff --git a/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala b/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala new file mode 100644 index 000000000..1a521686a --- /dev/null +++ b/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala @@ -0,0 +1,99 @@ +package io.udash +package rest.raw + +import com.avsystem.commons.annotation.explicitGenerics +import com.avsystem.commons.misc.ImplicitNotFound +import com.avsystem.commons.rpc.{AsRaw, AsRawReal, AsReal} +import com.avsystem.commons.serialization.GenCodec.ReadFailure +import monix.reactive.Observable + +import scala.annotation.implicitNotFound +import scala.reflect.{ClassTag, classTag} + +sealed trait StreamedBody { + final def defaultStatus: Int = this match { + case StreamedBody.Empty => 204 + case _ => 200 + } + + final def defaultResponse: StreamedRestResponse = + StreamedRestResponse( + code = defaultStatus, + headers = IMapping.empty, + body = this, + batchSize = 1, + ) +} +object StreamedBody extends StreamedBodyLowPrio { + case object Empty extends StreamedBody + + sealed trait NonEmpty extends StreamedBody { + def contentType: String + } + + /** TODO streaming doc */ + final case class RawBinary(content: Observable[Array[Byte]]) extends NonEmpty { + val contentType: String = HttpBody.OctetStreamType + + override def toString: String = super.toString + } + + /** TODO streaming doc */ + final case class JsonList( + elements: Observable[JsonValue], + charset: String = HttpBody.Utf8Charset, + ) extends NonEmpty { + val contentType: String = s"${HttpBody.JsonType};charset=$charset" + + override def toString: String = super.toString + } + + /** TODO streaming doc */ + final case class Single(body: HttpBody.NonEmpty) extends NonEmpty { + override def contentType: String = body.contentType + } + + def empty: StreamedBody = Empty + + def fromHttpBody(body: HttpBody): StreamedBody = body match { + case HttpBody.Empty => StreamedBody.Empty + case nonEmpty: HttpBody.NonEmpty => StreamedBody.Single(nonEmpty) + } + + def toHttpBody(body: StreamedBody): HttpBody = body match { + case StreamedBody.Empty => HttpBody.Empty + case nonEmpty: StreamedBody.NonEmpty => castOrFail[Single](nonEmpty).body + } + + @explicitGenerics + def castOrFail[T <: StreamedBody: ClassTag](body: StreamedBody): T = + body match { + case expected: T => expected + case unexpected => + throw new ReadFailure( + s"Expected ${classTag[T].runtimeClass.getSimpleName} body representation, got ${unexpected.getClass.getSimpleName}" + ) + } + + implicit val rawBinaryBodyForByteArray: AsRawReal[StreamedBody, Observable[Array[Byte]]] = + AsRawReal.create( + bytes => RawBinary(bytes), + body => StreamedBody.castOrFail[RawBinary](body).content, + ) +} +trait StreamedBodyLowPrio { this: StreamedBody.type => + implicit def bodyJsonListAsRaw[T](implicit jsonAsRaw: AsRaw[JsonValue, T]): AsRaw[StreamedBody, Observable[T]] = + v => StreamedBody.JsonList(v.map(jsonAsRaw.asRaw)) + implicit def bodyJsonListAsReal[T](implicit jsonAsReal: AsReal[JsonValue, T]): AsReal[StreamedBody, Observable[T]] = + v => StreamedBody.castOrFail[StreamedBody.JsonList](v).elements.map(jsonAsReal.asReal) + + @implicitNotFound("Cannot deserialize ${T} from StreamedBody, because:\n#{forJson}") + implicit def asRealNotFound[T]( + implicit forJson: ImplicitNotFound[AsReal[JsonValue, T]] + ): ImplicitNotFound[AsReal[StreamedBody, Observable[T]]] = ImplicitNotFound() + + @implicitNotFound("Cannot serialize ${T} into StreamedBody, because:\n#{forJson}") + implicit def asRawNotFound[T]( + implicit forJson: ImplicitNotFound[AsRaw[JsonValue, T]] + ): ImplicitNotFound[AsRaw[StreamedBody, Observable[T]]] = ImplicitNotFound() +} diff --git a/rest/src/main/scala/io/udash/rest/util/Utils.scala b/rest/src/main/scala/io/udash/rest/util/Utils.scala new file mode 100644 index 000000000..d9dc6a1fc --- /dev/null +++ b/rest/src/main/scala/io/udash/rest/util/Utils.scala @@ -0,0 +1,16 @@ +package io.udash +package rest.util + +import monix.eval.Task +import monix.reactive.Observable + +import java.io.ByteArrayOutputStream + +private[rest] object Utils { + + def mergeArrays(data: Observable[Array[Byte]]): Task[Array[Byte]] = + data.foldLeftL(new ByteArrayOutputStream()) { case (acc, elem) => + acc.write(elem) + acc + }.map(_.toByteArray) +} diff --git a/rest/src/test/scala/io/udash/rest/DirectRestApiTest.scala b/rest/src/test/scala/io/udash/rest/DirectRestApiTest.scala new file mode 100644 index 000000000..2f0ef10e0 --- /dev/null +++ b/rest/src/test/scala/io/udash/rest/DirectRestApiTest.scala @@ -0,0 +1,17 @@ +package io.udash +package rest + +import io.udash.rest.raw.{RawRest, RestRequest, RestResponse, StreamedRestResponse} +import monix.eval.Task + +class DirectRestApiTest extends RestApiTestScenarios with StreamingRestApiTestScenarios { + def clientHandle: RawRest.HandleRequest = serverHandle + + override def streamingClientHandler: RawRest.RestRequestHandler = new RawRest.RestRequestHandler { + override def handleRequest(request: RestRequest): Task[RestResponse] = + streamingServerHandle(request).map(_.asInstanceOf[RestResponse]) + + override def handleRequestStream(request: RestRequest): Task[StreamedRestResponse] = + streamingServerHandle(request).map(_.asInstanceOf[StreamedRestResponse]) + } +} diff --git a/rest/src/test/scala/io/udash/rest/RestApiTest.scala b/rest/src/test/scala/io/udash/rest/RestApiTest.scala index 50142a219..3ce504a95 100644 --- a/rest/src/test/scala/io/udash/rest/RestApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/RestApiTest.scala @@ -5,10 +5,10 @@ import cats.implicits.catsSyntaxTuple2Semigroupal import com.avsystem.commons.* import com.avsystem.commons.misc.ScalaDurationExtensions.durationIntOps import io.udash.rest.raw.RawRest -import io.udash.rest.raw.RawRest.HandleRequest import io.udash.testing.AsyncUdashSharedTest import monix.eval.Task import monix.execution.Scheduler +import monix.reactive.Observable import org.scalactic.source.Position import org.scalatest.time.{Millis, Seconds, Span} import org.scalatest.{Assertion, BeforeAndAfterEach} @@ -24,6 +24,7 @@ abstract class RestApiTest extends AsyncUdashSharedTest with BeforeAndAfterEach protected final val IdleTimout: FiniteDuration = CallTimeout * 100 protected val impl: RestTestApi.Impl = new RestTestApi.Impl + protected val streamingImpl: StreamingRestTestApi.Impl = new StreamingRestTestApi.Impl override protected def beforeEach(): Unit = { super.beforeEach() @@ -33,16 +34,30 @@ abstract class RestApiTest extends AsyncUdashSharedTest with BeforeAndAfterEach final val serverHandle: RawRest.HandleRequest = RawRest.asHandleRequest[RestTestApi](impl) + final val streamingServerHandle: RawRest.HandleRequestWithStreaming = + RawRest.asHandleRequestWithStreaming[StreamingRestTestApi](streamingImpl) + def clientHandle: RawRest.HandleRequest + def streamingClientHandler: RawRest.RestRequestHandler = + throw new UnsupportedOperationException(s"Streaming not supported in ${getClass.getSimpleName}") + lazy val proxy: RestTestApi = RawRest.fromHandleRequest[RestTestApi](clientHandle) + lazy val streamingProxy: StreamingRestTestApi = + RawRest.fromHandleRequestWithStreaming[StreamingRestTestApi](streamingClientHandler) + def testCall[T](call: RestTestApi => Future[T])(implicit pos: Position): Future[Assertion] = (call(proxy).wrapToTry, call(impl).catchFailures.wrapToTry).mapN { (proxyResult, implResult) => assert(proxyResult.map(mkDeep) == implResult.map(mkDeep)) } + def testStream[T](call: StreamingRestTestApi => Observable[T])(implicit pos: Position): Future[Assertion] = + (call(streamingProxy).toListL.materialize, call(streamingImpl).toListL.materialize).mapN { (proxyResult, implResult) => + assert(proxyResult.map(mkDeep) == implResult.map(mkDeep)) + }.runToFuture + def mkDeep(value: Any): Any = value match { case arr: Array[_] => IArraySeq.empty[AnyRef] ++ arr.iterator.map(mkDeep) case _ => value @@ -50,7 +65,8 @@ abstract class RestApiTest extends AsyncUdashSharedTest with BeforeAndAfterEach } trait RestApiTestScenarios extends RestApiTest { - override implicit val patienceConfig: PatienceConfig = PatienceConfig(scaled(Span(10, Seconds)), scaled(Span(50, Millis))) + override implicit val patienceConfig: PatienceConfig = + PatienceConfig(scaled(Span(10, Seconds)), scaled(Span(50, Millis))) "trivial GET" in { testCall(_.trivialGet) @@ -129,6 +145,14 @@ trait RestApiTestScenarios extends RestApiTest { } } -class DirectRestApiTest extends RestApiTestScenarios { - def clientHandle: HandleRequest = serverHandle +// TODO streaming MORE tests: cancellation, timeouts, errors, errors after sending a few elements, custom format, slow source observable +trait StreamingRestApiTestScenarios extends RestApiTest { + + "trivial GET stream" in { + testStream(_.simpleStream) + } + + "json GET stream" in { + testStream(_.jsonStream) + } } diff --git a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala new file mode 100644 index 000000000..28125ec23 --- /dev/null +++ b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala @@ -0,0 +1,25 @@ +package io.udash +package rest + +import monix.execution.Scheduler +import monix.reactive.Observable + +trait StreamingRestTestApi { + @GET def simpleStream: Observable[String] + + @GET def jsonStream: Observable[RestEntity] +} +object StreamingRestTestApi extends DefaultRestApiCompanion[StreamingRestTestApi] { + + import Scheduler.Implicits.global + + final class Impl extends StreamingRestTestApi { + override def simpleStream: Observable[String] = Observable("a", "b", "c") + + override def jsonStream: Observable[RestEntity] = Observable( + RestEntity(RestEntityId("1"), "first"), + RestEntity(RestEntityId("2"), "second"), + RestEntity(RestEntityId("3"), "third") + ) + } +} diff --git a/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala b/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala index 82881dee0..f120313e7 100644 --- a/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala +++ b/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala @@ -153,6 +153,8 @@ class RawRestTest extends AnyFunSuite with ScalaFutures { assert(future.futureValue == response) } + // TODO streaming add streaming tests + test("simple GET") { testRestCall(_.self.user(UserId("ID")), """-> GET /user?userId=ID diff --git a/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala b/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala index d8bc7e7f2..4bff878ec 100644 --- a/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala @@ -19,6 +19,8 @@ class ServerImplApiTest extends AnyFunSuite with ScalaFutures { assert(future.futureValue == response) } + // TODO streaming add streaming tests + test("simple GET call") { val params = RestParameters( path = PlainValue.decodePath("/thingy"), From 9c45195ccc055503b54e1751e002c79aec3a813c Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Thu, 27 Mar 2025 17:03:23 +0100 Subject: [PATCH 060/162] Fix CompilationErrorsTest --- .../io/udash/rest/CompilationErrorsTest.scala | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/rest/src/test/scala/io/udash/rest/CompilationErrorsTest.scala b/rest/src/test/scala/io/udash/rest/CompilationErrorsTest.scala index 61d798bb4..96d2f1ccd 100644 --- a/rest/src/test/scala/io/udash/rest/CompilationErrorsTest.scala +++ b/rest/src/test/scala/io/udash/rest/CompilationErrorsTest.scala @@ -17,6 +17,8 @@ class CompilationErrorsTest extends AnyFunSuite with CompilationErrorAssertions def meth(par: Any): Future[Unit] } + // TODO streaming add streaming tests + test("missing serializer for parameter") { val error = norm(typeErrorFor("object Api extends DefaultRestApiCompanion[MissingSerializerForParam]")) assert(error == @@ -41,7 +43,11 @@ class CompilationErrorsTest extends AnyFunSuite with CompilationErrorAssertions | Cannot serialize Any into RestResponse, because: | Cannot serialize Any into HttpBody, because: | Cannot serialize Any into JsonValue, because: - | No GenCodec found for Any""".stripMargin) + | No GenCodec found for Any + | * it cannot be translated into an HTTP stream method: + | scala.concurrent.Future[Any] is not a valid result type because: + | Cannot serialize Any into StreamedRestResponse, because: + | Cannot serialize Any into io.udash.rest.raw.StreamedBody, appropriate AsRaw instance not found""".stripMargin) } trait BadResultType { @@ -56,6 +62,8 @@ class CompilationErrorsTest extends AnyFunSuite with CompilationErrorAssertions | * it cannot be translated into a prefix method: | Unit is not a valid server REST API trait, does its companion extend DefaultRestApiCompanion, DefaultRestServerApiCompanion or other companion base? | * it cannot be translated into an HTTP method: + | Unit is not a valid result type of HTTP REST method - it must be a Future + | * it cannot be translated into an HTTP stream method: | Unit is not a valid result type of HTTP REST method - it must be a Future""".stripMargin) } @@ -71,6 +79,8 @@ class CompilationErrorsTest extends AnyFunSuite with CompilationErrorAssertions | * it cannot be translated into a prefix method: | prefix methods cannot take @Body parameters | * it cannot be translated into an HTTP method: + | CompilationErrorsTest.this.SubApi is not a valid result type of HTTP REST method - it must be a Future + | * it cannot be translated into an HTTP stream method: | CompilationErrorsTest.this.SubApi is not a valid result type of HTTP REST method - it must be a Future""".stripMargin) } @@ -84,7 +94,11 @@ class CompilationErrorsTest extends AnyFunSuite with CompilationErrorAssertions """cannot translate between trait UnexpectedGETBodyParam and trait RawRest: |problem with method meth: | * it cannot be translated into an HTTP GET method: - | GET methods cannot take @Body parameters""".stripMargin) + | GET methods cannot take @Body parameters + | * it cannot be translated into an HTTP GET stream method: + | scala.concurrent.Future[Unit] is not a valid result type because: + | Cannot serialize Unit into StreamedRestResponse, because: + | Cannot serialize Unit into io.udash.rest.raw.StreamedBody, appropriate AsRaw instance not found""".stripMargin) } trait MissingBodyParam { @@ -97,7 +111,11 @@ class CompilationErrorsTest extends AnyFunSuite with CompilationErrorAssertions """cannot translate between trait MissingBodyParam and trait RawRest: |problem with method meth: | * it cannot be translated into an HTTP method with custom body: - | expected exactly one @Body parameter but none was found""".stripMargin) + | expected exactly one @Body parameter but none was found + | * it cannot be translated into an HTTP stream method with custom body: + | scala.concurrent.Future[Unit] is not a valid result type because: + | Cannot serialize Unit into StreamedRestResponse, because: + | Cannot serialize Unit into io.udash.rest.raw.StreamedBody, appropriate AsRaw instance not found""".stripMargin) } trait MultipleBodyParams { @@ -110,6 +128,10 @@ class CompilationErrorsTest extends AnyFunSuite with CompilationErrorAssertions """cannot translate between trait MultipleBodyParams and trait RawRest: |problem with method meth: | * it cannot be translated into an HTTP method with custom body: - | expected exactly one @Body parameter but more than one was found""".stripMargin) + | expected exactly one @Body parameter but more than one was found + | * it cannot be translated into an HTTP stream method with custom body: + | scala.concurrent.Future[Unit] is not a valid result type because: + | Cannot serialize Unit into StreamedRestResponse, because: + | Cannot serialize Unit into io.udash.rest.raw.StreamedBody, appropriate AsRaw instance not found""".stripMargin) } } From 56f7db20dd0ce758171a7ec789ffe7fe5a2629b7 Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Fri, 28 Mar 2025 15:06:34 +0100 Subject: [PATCH 061/162] Handle empty observable result --- .../scala/io/udash/rest/RestServlet.scala | 17 +++--- .../test/resources/StreamingRestTestApi.json | 21 +++++++ .../io/udash/rest/jetty/JettyRestClient.scala | 61 +++++++++---------- .../scala/io/udash/rest/RestApiTest.scala | 4 ++ .../io/udash/rest/StreamingRestTestApi.scala | 4 ++ 5 files changed, 68 insertions(+), 39 deletions(-) diff --git a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala index 39433d6c8..f359701f4 100644 --- a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala +++ b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala @@ -1,22 +1,22 @@ package io.udash package rest -import com.avsystem.commons._ +import com.avsystem.commons.* import com.avsystem.commons.annotation.explicitGenerics import com.typesafe.scalalogging.LazyLogging -import io.udash.rest.RestServlet._ -import io.udash.rest.raw._ +import io.udash.rest.RestServlet.* +import io.udash.rest.raw.* import io.udash.utils.URLEncoder import monix.eval.Task import monix.execution.Scheduler -import monix.reactive.Consumer +import monix.reactive.{Consumer, Observable} import java.io.ByteArrayOutputStream import java.util.concurrent.atomic.AtomicBoolean import javax.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} import javax.servlet.{AsyncEvent, AsyncListener} import scala.annotation.tailrec -import scala.concurrent.duration._ +import scala.concurrent.duration.* object RestServlet { final val DefaultHandleTimeout = 30.seconds @@ -124,20 +124,21 @@ class RestServlet( case jsonList: StreamedBody.JsonList => // TODO streaming document no content length behaviour in relation to the client response.setContentType(jsonList.contentType) - response.getOutputStream.write("[".getBytes(jsonList.charset)) jsonList.elements .bufferTumbling(stream.batchSize) + .switchIfEmpty(Observable(Seq.empty)) .zipWithIndex .consumeWith(Consumer.foreach { case (batch, idx) => val firstBatch = idx == 0 - if (firstBatch) + if (firstBatch) { + response.getOutputStream.write("[".getBytes(jsonList.charset)) batch.iterator.zipWithIndex.foreach { case (e, idx) => if (idx != 0) { response.getOutputStream.write(",".getBytes(jsonList.charset)) } response.getOutputStream.write(e.value.getBytes(jsonList.charset)) } - else + } else batch.foreach { e => response.getOutputStream.write(",".getBytes(jsonList.charset)) response.getOutputStream.write(e.value.getBytes(jsonList.charset)) diff --git a/rest/.jvm/src/test/resources/StreamingRestTestApi.json b/rest/.jvm/src/test/resources/StreamingRestTestApi.json index 39cdb5407..5204c4ef0 100644 --- a/rest/.jvm/src/test/resources/StreamingRestTestApi.json +++ b/rest/.jvm/src/test/resources/StreamingRestTestApi.json @@ -6,6 +6,27 @@ "description": "Some test REST API" }, "paths": { + "/emptyStream": { + "get": { + "operationId": "emptyStream", + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + } + } + } + } + } + } + }, "/jsonStream": { "get": { "operationId": "jsonStream", diff --git a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala index 5b81b815d..2ef198f8d 100644 --- a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala +++ b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala @@ -99,41 +99,40 @@ final class JettyRestClient( } } - override def onFailure(response: Response, failure: Throwable): Unit = { - super.onFailure(response, failure) - // TODO streaming error handling client-side ??? - } - override def onComplete(result: Result): Unit = { super.onComplete(result) - val httpResp = result.getResponse - val contentLength = httpResp.getHeaders.getLongField(HttpHeader.CONTENT_LENGTH) - if (contentLength != -1) { - val contentTypeOpt = httpResp.getHeaders.get(HttpHeader.CONTENT_TYPE).opt - val charsetOpt = contentTypeOpt.map(MimeTypes.getCharsetFromContentType) - // TODO streaming client-side handle errors ? - val rawBody = getInputStream.readAllBytes() - val body = (contentTypeOpt, charsetOpt) match { - case (Opt(contentType), Opt(charset)) => - StreamedBody.fromHttpBody( - HttpBody.textual( - content = new String(rawBody, charset), - mediaType = MimeTypes.getContentTypeWithoutCharset(contentType), - charset = charset, + if (result.isSucceeded) { + val httpResp = result.getResponse + val contentLength = httpResp.getHeaders.getLongField(HttpHeader.CONTENT_LENGTH) + if (contentLength != -1) { + val contentTypeOpt = httpResp.getHeaders.get(HttpHeader.CONTENT_TYPE).opt + val charsetOpt = contentTypeOpt.map(MimeTypes.getCharsetFromContentType) + // TODO streaming client-side handle errors ? + val rawBody = getInputStream.readAllBytes() + val body = (contentTypeOpt, charsetOpt) match { + case (Opt(contentType), Opt(charset)) => + StreamedBody.fromHttpBody( + HttpBody.textual( + content = new String(rawBody, charset), + mediaType = MimeTypes.getContentTypeWithoutCharset(contentType), + charset = charset, + ) ) - ) - case (Opt(contentType), Opt.Empty) => - StreamedBody.fromHttpBody(HttpBody.binary(rawBody, contentType)) - case _ => - StreamedBody.Empty + case (Opt(contentType), Opt.Empty) => + StreamedBody.fromHttpBody(HttpBody.binary(rawBody, contentType)) + case _ => + StreamedBody.Empty + } + val restResponse = StreamedRestResponse( + code = httpResp.getStatus, + headers = parseHeaders(httpResp), + body = body, + batchSize = 1, + ) + callback(Success(restResponse)) } - val restResponse = StreamedRestResponse( - code = httpResp.getStatus, - headers = parseHeaders(httpResp), - body = body, - batchSize = 1, - ) - callback(Success(restResponse)) + } else { + callback(Failure(result.getFailure)) } } } diff --git a/rest/src/test/scala/io/udash/rest/RestApiTest.scala b/rest/src/test/scala/io/udash/rest/RestApiTest.scala index 3ce504a95..c7573f9ca 100644 --- a/rest/src/test/scala/io/udash/rest/RestApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/RestApiTest.scala @@ -148,6 +148,10 @@ trait RestApiTestScenarios extends RestApiTest { // TODO streaming MORE tests: cancellation, timeouts, errors, errors after sending a few elements, custom format, slow source observable trait StreamingRestApiTestScenarios extends RestApiTest { + "empty GET stream" in { + testStream(_.emptyStream) + } + "trivial GET stream" in { testStream(_.simpleStream) } diff --git a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala index 28125ec23..863987ba0 100644 --- a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala @@ -5,6 +5,8 @@ import monix.execution.Scheduler import monix.reactive.Observable trait StreamingRestTestApi { + @GET def emptyStream: Observable[Int] + @GET def simpleStream: Observable[String] @GET def jsonStream: Observable[RestEntity] @@ -14,6 +16,8 @@ object StreamingRestTestApi extends DefaultRestApiCompanion[StreamingRestTestApi import Scheduler.Implicits.global final class Impl extends StreamingRestTestApi { + override def emptyStream: Observable[Int] = Observable.empty + override def simpleStream: Observable[String] = Observable("a", "b", "c") override def jsonStream: Observable[RestEntity] = Observable( From aba85da6a394f94db7018afd52554b784bcab4ce Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Fri, 28 Mar 2025 16:11:35 +0100 Subject: [PATCH 062/162] Adjust tests --- .../test/resources/StreamingRestTestApi.json | 42 ++++++++++++++++--- .../scala/io/udash/rest/RestApiTest.scala | 4 +- .../io/udash/rest/StreamingRestTestApi.scala | 20 ++++++--- 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/rest/.jvm/src/test/resources/StreamingRestTestApi.json b/rest/.jvm/src/test/resources/StreamingRestTestApi.json index 5204c4ef0..27d25a66f 100644 --- a/rest/.jvm/src/test/resources/StreamingRestTestApi.json +++ b/rest/.jvm/src/test/resources/StreamingRestTestApi.json @@ -6,9 +6,27 @@ "description": "Some test REST API" }, "paths": { - "/emptyStream": { - "get": { - "operationId": "emptyStream", + "/errorStream": { + "post": { + "operationId": "errorStream", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "immediate": { + "type": "boolean" + } + }, + "required": [ + "immediate" + ] + } + } + }, + "required": true + }, "responses": { "200": { "description": "Success", @@ -17,8 +35,7 @@ "schema": { "type": "array", "items": { - "type": "integer", - "format": "int32" + "$ref": "#/components/schemas/RestEntity" } } } @@ -50,6 +67,18 @@ "/simpleStream": { "get": { "operationId": "simpleStream", + "parameters": [ + { + "name": "size", + "in": "query", + "required": true, + "explode": false, + "schema": { + "type": "integer", + "format": "int32" + } + } + ], "responses": { "200": { "description": "Success", @@ -58,7 +87,8 @@ "schema": { "type": "array", "items": { - "type": "string" + "type": "integer", + "format": "int32" } } } diff --git a/rest/src/test/scala/io/udash/rest/RestApiTest.scala b/rest/src/test/scala/io/udash/rest/RestApiTest.scala index c7573f9ca..ee777aed2 100644 --- a/rest/src/test/scala/io/udash/rest/RestApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/RestApiTest.scala @@ -149,11 +149,11 @@ trait RestApiTestScenarios extends RestApiTest { trait StreamingRestApiTestScenarios extends RestApiTest { "empty GET stream" in { - testStream(_.emptyStream) + testStream(_.simpleStream(0)) } "trivial GET stream" in { - testStream(_.simpleStream) + testStream(_.simpleStream(5)) } "json GET stream" in { diff --git a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala index 863987ba0..87730a13d 100644 --- a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala @@ -1,29 +1,39 @@ package io.udash package rest +import io.udash.rest.raw.HttpErrorException import monix.execution.Scheduler import monix.reactive.Observable trait StreamingRestTestApi { - @GET def emptyStream: Observable[Int] - - @GET def simpleStream: Observable[String] + @GET def simpleStream(size: Int): Observable[Int] @GET def jsonStream: Observable[RestEntity] + + @POST def errorStream(immediate: Boolean): Observable[RestEntity] } object StreamingRestTestApi extends DefaultRestApiCompanion[StreamingRestTestApi] { import Scheduler.Implicits.global final class Impl extends StreamingRestTestApi { - override def emptyStream: Observable[Int] = Observable.empty - override def simpleStream: Observable[String] = Observable("a", "b", "c") + override def simpleStream(size: Int): Observable[Int] = + Observable.fromIterable(Range(0, size)) override def jsonStream: Observable[RestEntity] = Observable( RestEntity(RestEntityId("1"), "first"), RestEntity(RestEntityId("2"), "second"), RestEntity(RestEntityId("3"), "third") ) + + override def errorStream(immediate: Boolean): Observable[RestEntity] = + if (immediate) + Observable.raiseError(HttpErrorException.plain(400, "bad")) + else + Observable.fromIterable(Range(0, 3)).map { i => + if (i < 2) RestEntity(RestEntityId(i.toString), "first") + else throw HttpErrorException.plain(400, "later bad") + } } } From d27c904be921db63d9b45c63608536bb40046893 Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Fri, 28 Mar 2025 18:41:55 +0100 Subject: [PATCH 063/162] Rework client impl to handle errors --- .../test/resources/StreamingRestTestApi.json | 45 +++++--- .../io/udash/rest/jetty/JettyRestClient.scala | 106 ++++++++++-------- .../udash/rest/jetty/JettyRestCallTest.scala | 4 +- .../scala/io/udash/rest/RestApiTest.scala | 15 ++- .../io/udash/rest/StreamingRestTestApi.scala | 9 +- 5 files changed, 112 insertions(+), 67 deletions(-) diff --git a/rest/.jvm/src/test/resources/StreamingRestTestApi.json b/rest/.jvm/src/test/resources/StreamingRestTestApi.json index 27d25a66f..51bfc9329 100644 --- a/rest/.jvm/src/test/resources/StreamingRestTestApi.json +++ b/rest/.jvm/src/test/resources/StreamingRestTestApi.json @@ -6,27 +6,38 @@ "description": "Some test REST API" }, "paths": { + "/binaryStream": { + "post": { + "operationId": "binaryStream", + "responses": { + "200": { + "description": "Success", + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + } + } + } + } + }, "/errorStream": { "post": { "operationId": "errorStream", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "immediate": { - "type": "boolean" - } - }, - "required": [ - "immediate" - ] - } + "parameters": [ + { + "name": "immediate", + "in": "query", + "required": true, + "explode": false, + "schema": { + "type": "boolean" } - }, - "required": true - }, + } + ], "responses": { "200": { "description": "Success", diff --git a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala index 2ef198f8d..f75733db8 100644 --- a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala +++ b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala @@ -8,10 +8,13 @@ import io.udash.rest.raw.* import io.udash.rest.util.Utils import io.udash.utils.URLEncoder import monix.eval.Task -import monix.execution.Callback -import monix.reactive.Observable +import monix.execution.{Callback, Scheduler} +import monix.reactive.OverflowStrategy.Unbounded +import monix.reactive.{MulticastStrategy, Observable} +import monix.reactive.subjects.{ConcurrentSubject, PublishToOneSubject} import org.eclipse.jetty.client.* import org.eclipse.jetty.http.{HttpCookie, HttpHeader, MimeTypes} +import org.eclipse.jetty.io.Content import java.nio.charset.Charset import scala.concurrent.CancellationException @@ -48,8 +51,12 @@ final class JettyRestClient( override def handleRequestStream(request: RestRequest): Task[StreamedRestResponse] = prepareRequest(baseUrl, timeout, request).flatMap { httpReq => - Task.async { (callback: Callback[Throwable, StreamedRestResponse]) => - val listener = new InputStreamResponseListener { + Task.async0 { (scheduler: Scheduler, callback: Callback[Throwable, StreamedRestResponse]) => + val listener = new BufferingResponseListener(maxResponseLength) { + private var collectToBuffer: Boolean = true + private lazy val publishSubject = PublishToOneSubject[Array[Byte]]() + private lazy val rawContentSubject = ConcurrentSubject.from(publishSubject, Unbounded)(scheduler) + override def onHeaders(response: Response): Unit = { super.onHeaders(response) // TODO streaming document content length behaviour @@ -57,27 +64,22 @@ final class JettyRestClient( if (contentLength == -1) { val contentTypeOpt = response.getHeaders.get(HttpHeader.CONTENT_TYPE).opt val mediaTypeOpt = contentTypeOpt.map(MimeTypes.getContentTypeWithoutCharset) - val charsetOpt = contentTypeOpt.map(MimeTypes.getCharsetFromContentType) - // TODO streaming error handling client-side ??? val bodyOpt = mediaTypeOpt matchOpt { case Opt(HttpBody.OctetStreamType) => - // TODO streaming configure chunk size ??? - StreamedBody.RawBinary(Observable.fromInputStream(Task.eval(getInputStream))) + StreamedBody.RawBinary(content = rawContentSubject) case Opt(HttpBody.JsonType) => - val charset = charsetOpt.getOrElse(HttpBody.Utf8Charset) + val charset = contentTypeOpt.map(MimeTypes.getCharsetFromContentType).getOrElse(HttpBody.Utf8Charset) // suboptimal - maybe "online" parsing is possible using Jackson / other lib without waiting for full content ? - val elements: Observable[JsonValue] = - Observable - .fromTask(Utils.mergeArrays(Observable.fromInputStream(Task.eval(getInputStream)))) + StreamedBody.JsonList( + elements = Observable + .fromTask(Utils.mergeArrays(rawContentSubject)) .map(raw => new String(raw, charset)) .flatMap { jsonStr => val input = new JsonStringInput(new JsonReader(jsonStr)) Observable .fromIterator(Task.eval(input.readList().iterator(_.asInstanceOf[JsonStringInput].readRawJson()))) .map(JsonValue(_)) - } - StreamedBody.JsonList( - elements = elements, + }, charset = charset, ) } @@ -87,6 +89,7 @@ final class JettyRestClient( callback(Failure(new Exception(s"Unsupported content type $contentTypeOpt"))) }, body => { + this.collectToBuffer = false val restResponse = StreamedRestResponse( code = response.getStatus, headers = parseHeaders(response), @@ -99,42 +102,40 @@ final class JettyRestClient( } } - override def onComplete(result: Result): Unit = { - super.onComplete(result) + override def onContent(response: Response, chunk: Content.Chunk, demander: Runnable): Unit = + if (collectToBuffer) + super.onContent(response, chunk, demander) + else + if (chunk == Content.Chunk.EOF) { + rawContentSubject.onComplete() + } else { + val buf = chunk.getByteBuffer + val arr = new Array[Byte](buf.remaining) + buf.get(arr) + publishSubject.subscription // wait for subscription + .flatMapNow(_ => rawContentSubject.onNext(arr)) + .mapNow(_ => demander.run()) + } + + override def onComplete(result: Result): Unit = if (result.isSucceeded) { val httpResp = result.getResponse val contentLength = httpResp.getHeaders.getLongField(HttpHeader.CONTENT_LENGTH) if (contentLength != -1) { - val contentTypeOpt = httpResp.getHeaders.get(HttpHeader.CONTENT_TYPE).opt - val charsetOpt = contentTypeOpt.map(MimeTypes.getCharsetFromContentType) // TODO streaming client-side handle errors ? - val rawBody = getInputStream.readAllBytes() - val body = (contentTypeOpt, charsetOpt) match { - case (Opt(contentType), Opt(charset)) => - StreamedBody.fromHttpBody( - HttpBody.textual( - content = new String(rawBody, charset), - mediaType = MimeTypes.getContentTypeWithoutCharset(contentType), - charset = charset, - ) - ) - case (Opt(contentType), Opt.Empty) => - StreamedBody.fromHttpBody(HttpBody.binary(rawBody, contentType)) - case _ => - StreamedBody.Empty - } val restResponse = StreamedRestResponse( code = httpResp.getStatus, headers = parseHeaders(httpResp), - body = body, + body = StreamedBody.fromHttpBody(parseHttpBody(httpResp, this)), batchSize = 1, ) callback(Success(restResponse)) + } else { + rawContentSubject.onComplete() } } else { callback(Failure(result.getFailure)) } - } } httpReq.send(listener) }.doOnCancel(Task(httpReq.abort(new CancellationException("Request cancelled")))) @@ -193,17 +194,11 @@ final class JettyRestClient( override def onComplete(result: Result): Unit = if (result.isSucceeded) { val httpResp = result.getResponse - val contentTypeOpt = httpResp.getHeaders.get(HttpHeader.CONTENT_TYPE).opt - val charsetOpt = contentTypeOpt.map(MimeTypes.getCharsetFromContentType) - val body = (contentTypeOpt, charsetOpt) match { - case (Opt(contentType), Opt(charset)) => - HttpBody.textual(getContentAsString, MimeTypes.getContentTypeWithoutCharset(contentType), charset) - case (Opt(contentType), Opt.Empty) => - HttpBody.binary(getContent, contentType) - case _ => - HttpBody.Empty - } - val response = RestResponse(httpResp.getStatus, parseHeaders(httpResp), body) + val response = RestResponse( + code = httpResp.getStatus, + headers = parseHeaders(httpResp), + body = parseHttpBody(httpResp, this), + ) callback(Success(response)) } else { callback(Failure(result.getFailure)) @@ -212,6 +207,23 @@ final class JettyRestClient( } .doOnCancel(Task(httpReq.abort(new CancellationException("Request cancelled")))) + private def parseHttpBody(httpResp: Response, listener: BufferingResponseListener): HttpBody = { + val contentTypeOpt = httpResp.getHeaders.get(HttpHeader.CONTENT_TYPE).opt + val charsetOpt = contentTypeOpt.map(MimeTypes.getCharsetFromContentType) + (contentTypeOpt, charsetOpt) match { + case (Opt(contentType), Opt(charset)) => + HttpBody.textual( + content = listener.getContentAsString, + mediaType = MimeTypes.getContentTypeWithoutCharset(contentType), + charset = charset, + ) + case (Opt(contentType), Opt.Empty) => + HttpBody.binary(listener.getContent, contentType) + case _ => + HttpBody.Empty + } + } + private def parseHeaders(httpResp: Response): IMapping[PlainValue] = IMapping(httpResp.getHeaders.asScala.iterator.map(h => (h.getName, PlainValue(h.getValue))).toList) } diff --git a/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala b/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala index b23793f72..32c70b071 100644 --- a/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala +++ b/rest/jetty/src/test/scala/io/udash/rest/jetty/JettyRestCallTest.scala @@ -6,7 +6,9 @@ import io.udash.rest.{RestApiTestScenarios, ServletBasedRestApiTest, StreamingRe import org.eclipse.jetty.client.HttpClient final class JettyRestCallTest - extends ServletBasedRestApiTest with RestApiTestScenarios with StreamingRestApiTestScenarios { + extends ServletBasedRestApiTest + with RestApiTestScenarios + with StreamingRestApiTestScenarios { /** * Similar to the default HttpClient, but with a connection timeout diff --git a/rest/src/test/scala/io/udash/rest/RestApiTest.scala b/rest/src/test/scala/io/udash/rest/RestApiTest.scala index ee777aed2..ceb59b35b 100644 --- a/rest/src/test/scala/io/udash/rest/RestApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/RestApiTest.scala @@ -55,7 +55,7 @@ abstract class RestApiTest extends AsyncUdashSharedTest with BeforeAndAfterEach def testStream[T](call: StreamingRestTestApi => Observable[T])(implicit pos: Position): Future[Assertion] = (call(streamingProxy).toListL.materialize, call(streamingImpl).toListL.materialize).mapN { (proxyResult, implResult) => - assert(proxyResult.map(mkDeep) == implResult.map(mkDeep)) + assert(proxyResult.map(_.map(mkDeep)) == implResult.map(_.map(mkDeep))) }.runToFuture def mkDeep(value: Any): Any = value match { @@ -159,4 +159,17 @@ trait StreamingRestApiTestScenarios extends RestApiTest { "json GET stream" in { testStream(_.jsonStream) } + + "binary stream" in { + testStream(_.binaryStream()) + } + + "immediate stream error" in { + testStream(_.errorStream(immediate = true)) + } + + // TODO streaming - does not work on client side + "mid-stream error" ignore { + testStream(_.errorStream(immediate = false)) + } } diff --git a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala index 87730a13d..606eb2b0f 100644 --- a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala @@ -5,12 +5,16 @@ import io.udash.rest.raw.HttpErrorException import monix.execution.Scheduler import monix.reactive.Observable +import scala.concurrent.duration._ + trait StreamingRestTestApi { @GET def simpleStream(size: Int): Observable[Int] @GET def jsonStream: Observable[RestEntity] - @POST def errorStream(immediate: Boolean): Observable[RestEntity] + @POST def binaryStream(): Observable[Array[Byte]] + + @POST def errorStream(@Query immediate: Boolean): Observable[RestEntity] } object StreamingRestTestApi extends DefaultRestApiCompanion[StreamingRestTestApi] { @@ -27,6 +31,9 @@ object StreamingRestTestApi extends DefaultRestApiCompanion[StreamingRestTestApi RestEntity(RestEntityId("3"), "third") ) + override def binaryStream(): Observable[Array[Byte]] = + Observable("abc".getBytes, "xyz".getBytes) + override def errorStream(immediate: Boolean): Observable[RestEntity] = if (immediate) Observable.raiseError(HttpErrorException.plain(400, "bad")) From 3ee14b485ba87b2b3538ff8c088f183efc440615 Mon Sep 17 00:00:00 2001 From: Krzysztof Maliszewski Date: Wed, 2 Apr 2025 13:28:12 +0200 Subject: [PATCH 064/162] Enhance documentation for streaming support --- .../scala/io/udash/rest/RestServlet.scala | 6 ++- .../io/udash/rest/jetty/JettyRestClient.scala | 42 ++++++++++++++++--- .../scala/io/udash/rest/raw/RawRest.scala | 6 ++- .../io/udash/rest/raw/RestResponse.scala | 22 ++++++---- .../io/udash/rest/raw/StreamedBody.scala | 18 ++++++-- 5 files changed, 75 insertions(+), 19 deletions(-) diff --git a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala index f359701f4..ddad45eb3 100644 --- a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala +++ b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala @@ -111,18 +111,20 @@ class RestServlet( stream: StreamedRestResponse, body: StreamedBody.NonEmpty, ): Task[Unit] = Task.defer { + // The Content-Length header is intentionally omitted for streams. + // This signals to the client that the response body size is not predetermined and will be streamed. + // Clients implementing the streaming part of the REST interface contract MUST be prepared + // to handle responses without Content-Length by reading data incrementally until the stream completes. body match { case single: StreamedBody.Single => Task.eval(writeNonEmptyBody(response, single.body)) case binary: StreamedBody.RawBinary => - // TODO streaming document no content length behaviour in relation to the client response.setContentType(binary.contentType) binary.content.bufferTumbling(stream.batchSize).consumeWith(Consumer.foreach { batch => batch.foreach(e => response.getOutputStream.write(e)) response.getOutputStream.flush() }) case jsonList: StreamedBody.JsonList => - // TODO streaming document no content length behaviour in relation to the client response.setContentType(jsonList.contentType) jsonList.elements .bufferTumbling(stream.batchSize) diff --git a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala index f75733db8..6d2b9f6cf 100644 --- a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala +++ b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala @@ -20,8 +20,18 @@ import java.nio.charset.Charset import scala.concurrent.CancellationException import scala.concurrent.duration.* -/** TODO streaming doc */ -final class JettyRestClient( +/** + * A REST client implementation based on the Eclipse Jetty HTTP client library. + * Supports both standard request/response interactions and handling of streamed responses. + * + * Streaming responses allow processing large amounts of data without buffering the entire + * response body in memory. This client activates streaming mode *only* when the server's + * response headers *do not* include a `Content-Length`. + * + * @param client The configured Jetty `HttpClient` instance. + * @param defaultMaxResponseLength Default maximum size (in bytes) for buffering non-streamed responses. + * @param defaultTimeout Default timeout for requests. + */final class JettyRestClient( client: HttpClient, defaultMaxResponseLength: Int = JettyRestClient.DefaultMaxResponseLength, defaultTimeout: Duration = JettyRestClient.DefaultTimeout, @@ -37,7 +47,16 @@ final class JettyRestClient( asHandleRequestWithStreaming(baseUri, customMaxResponseLength, customTimeout) ) - /** TODO streaming doc */ + /** + * Creates a request handler with streaming support that can be used to make REST calls. + * The handler supports both regular responses and streaming responses, allowing for + * incremental processing of large payloads through Observable streams. + * + * @param baseUrl Base URL for the REST service + * @param customMaxResponseLength Optional maximum response length override for non-streamed responses + * @param customTimeout Optional timeout override + * @return A handler that can process REST requests with streaming capabilities + */ def asHandleRequestWithStreaming( baseUrl: String, customMaxResponseLength: OptArg[Int] = OptArg.Empty, @@ -59,7 +78,9 @@ final class JettyRestClient( override def onHeaders(response: Response): Unit = { super.onHeaders(response) - // TODO streaming document content length behaviour + // When Content-Length is not provided (-1), process the response as a stream + // since we can't determine the full size in advance. This enables handling + // chunked transfer encoding and streaming responses. val contentLength = response.getHeaders.getLongField(HttpHeader.CONTENT_LENGTH) if (contentLength == -1) { val contentTypeOpt = response.getHeaders.get(HttpHeader.CONTENT_TYPE).opt @@ -122,7 +143,8 @@ final class JettyRestClient( val httpResp = result.getResponse val contentLength = httpResp.getHeaders.getLongField(HttpHeader.CONTENT_LENGTH) if (contentLength != -1) { - // TODO streaming client-side handle errors ? + // For responses with known content length, we handle them as regular (non-streamed) responses + // Any errors will be propagated through the callback's Failure channel val restResponse = StreamedRestResponse( code = httpResp.getStatus, headers = parseHeaders(httpResp), @@ -142,7 +164,15 @@ final class JettyRestClient( } } - /** TODO streaming doc */ + /** + * Creates a `RawRest.HandleRequest` which handles standard REST requests by buffering the entire response. + * This does *not* support streaming responses. + * + * @param baseUrl The base URL for the REST service. + * @param customMaxResponseLength Optional override for the maximum response length. + * @param customTimeout Optional override for the request timeout. + * @return A `RawRest.HandleRequest` that buffers responses. + */ def asHandleRequest( baseUrl: String, customMaxResponseLength: OptArg[Int] = OptArg.Empty, diff --git a/rest/src/main/scala/io/udash/rest/raw/RawRest.scala b/rest/src/main/scala/io/udash/rest/raw/RawRest.scala index 61d247b3b..776809b26 100644 --- a/rest/src/main/scala/io/udash/rest/raw/RawRest.scala +++ b/rest/src/main/scala/io/udash/rest/raw/RawRest.scala @@ -141,7 +141,11 @@ trait RawRest { def asHandleRequestWithStreaming(metadata: RestMetadata[_]): HandleRequestWithStreaming = RawRest.resolveAndHandle(metadata)(handleResolvedWithStreaming) - // TODO doc for compatibility + /** + * Handles a resolved REST call and returns a standard RestResponse. + * This method is maintained for backward compatibility with non-streaming clients. + * It delegates to handleResolvedWithStreaming and converts any streaming response to a standard response. + */ def handleResolved(request: RestRequest, resolved: ResolvedCall): Task[RestResponse] = StreamedRestResponse.fallbackToRestResponse(handleResolvedWithStreaming(request, resolved)) diff --git a/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala b/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala index 806d12813..f0709d59e 100644 --- a/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala +++ b/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala @@ -1,18 +1,17 @@ package io.udash package rest.raw -import com.avsystem.commons._ +import com.avsystem.commons.* import com.avsystem.commons.misc.ImplicitNotFound import com.avsystem.commons.rpc.{AsRaw, AsReal} import io.udash.rest.raw.RawRest.FromTask -import io.udash.rest.raw.StreamedBody.castOrFail import io.udash.rest.util.Utils import monix.eval.{Task, TaskLike} import monix.reactive.Observable import scala.annotation.implicitNotFound -/** TODO streaming doc */ +/** Base trait for REST response types, either standard or streaming. Contains common properties like status code and headers. */ sealed trait AbstractRestResponse { def code: Int def headers: IMapping[PlainValue] @@ -20,7 +19,7 @@ sealed trait AbstractRestResponse { final def isSuccess: Boolean = code >= 200 && code < 300 } -/** TODO streaming doc */ +/** Standard REST response containing a status code, headers, and a body. The body is loaded fully in memory as an HttpBody. */ final case class RestResponse( code: Int, headers: IMapping[PlainValue], @@ -112,7 +111,10 @@ trait RestResponseLowPrio { this: RestResponse.type => ): ImplicitNotFound[AsRaw[RestResponse, T]] = ImplicitNotFound() } -/** TODO streaming doc */ +/** + * Streaming REST response containing a status code, headers, and a streamed body. + * Unlike standard RestResponse, the body content can be delivered incrementally through a reactive stream. + */ final case class StreamedRestResponse( code: Int, headers: IMapping[PlainValue], @@ -129,7 +131,10 @@ final case class StreamedRestResponse( object StreamedRestResponse extends StreamedRestResponseLowPrio { - /** TODO streaming doc */ + /** + * Converts a StreamedRestResponse to a standard RestResponse by materializing streamed content. + * This is useful for compatibility with APIs that don't support streaming. + */ def fallbackToRestResponse(response: StreamedRestResponse): Task[RestResponse] = { val httpBody: Task[HttpBody] = response.body match { case StreamedBody.Empty => @@ -152,7 +157,10 @@ object StreamedRestResponse extends StreamedRestResponseLowPrio { httpBody.map(RestResponse(response.code, response.headers, _)) } - /** TODO doc */ + /** + * Converts any AbstractRestResponse to a standard RestResponse by materializing streamed content if necessary. + * This is useful for compatibility with APIs that don't support streaming. + */ def fallbackToRestResponse(response: Task[AbstractRestResponse]): Task[RestResponse] = response.flatMap { case restResponse: RestResponse => Task.now(restResponse) diff --git a/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala b/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala index 1a521686a..e39193815 100644 --- a/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala +++ b/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala @@ -31,14 +31,22 @@ object StreamedBody extends StreamedBodyLowPrio { def contentType: String } - /** TODO streaming doc */ + /** + * Represents a binary streamed response body. + * The content is delivered as a stream of byte arrays which can be processed incrementally. + * Useful for large binary files or content that is generated dynamically. + */ final case class RawBinary(content: Observable[Array[Byte]]) extends NonEmpty { val contentType: String = HttpBody.OctetStreamType override def toString: String = super.toString } - /** TODO streaming doc */ + /** + * Represents a streamed list of JSON values. + * Each element in the stream is a complete JSON value, allowing for incremental processing + * of potentially large collections without loading everything into memory at once. + */ final case class JsonList( elements: Observable[JsonValue], charset: String = HttpBody.Utf8Charset, @@ -48,7 +56,11 @@ object StreamedBody extends StreamedBodyLowPrio { override def toString: String = super.toString } - /** TODO streaming doc */ + /** + * Represents a single non-empty HTTP body that will be delivered as a streaming response. + * Used when the content is already fully loaded but needs to be returned through a streaming API + * for consistency with other streaming operations. + */ final case class Single(body: HttpBody.NonEmpty) extends NonEmpty { override def contentType: String = body.contentType } From c56f6b8293746ff115150735ff10cb1567ed6f58 Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Wed, 2 Apr 2025 17:58:07 +0200 Subject: [PATCH 065/162] Group Jetty Scala Steward update PRs --- .scala-steward.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.scala-steward.conf b/.scala-steward.conf index 973289993..3d862779e 100644 --- a/.scala-steward.conf +++ b/.scala-steward.conf @@ -4,3 +4,10 @@ updates.ignore = [ updates.pin = [ {groupId = "ch.qos.logback", version = "1.3."}, ] +pullRequests.grouping = [ + { + name = "jetty", + title = "Update Jetty dependencies", + filter = [{ group = "org.eclipse.jetty" }, { group = "org.eclipse.jetty*" }] + } +] From e90d5f995033a9978fc53493da68ac4c93bc06eb Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Wed, 2 Apr 2025 18:01:27 +0200 Subject: [PATCH 066/162] Update selenium-java to 4.30.0 (#1339) * Update selenium-java to 4.30.0 * Remove extra newline in Dependencies.scala --------- Co-authored-by: Dawid Dworak --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 1a92b6f0d..b5c7c5266 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -42,7 +42,7 @@ object Dependencies { val bootstrap4DatepickerVersion = "5.39.0" val momentJsVersion = "2.30.1" - val seleniumVersion = "4.29.0" + val seleniumVersion = "4.30.0" val webDriverManagerVersion = "6.0.0" val scalaJsBenchmarkVersion = "0.10.0" From f94707220d2340b271c52d2715b1d41161814771 Mon Sep 17 00:00:00 2001 From: Krzysztof Maliszewski Date: Sat, 5 Apr 2025 11:14:32 +0200 Subject: [PATCH 067/162] Add streaming support testing to REST API with new endpoints and error handling --- .../test/resources/StreamingRestTestApi.json | 43 ++++ .../io/udash/rest/jetty/JettyRestClient.scala | 2 +- .../main/scala/io/udash/rest/companions.scala | 3 + .../scala/io/udash/rest/raw/RestRequest.scala | 5 +- .../scala/io/udash/rest/RestApiTest.scala | 30 ++- .../io/udash/rest/SomeServerApiImpl.scala | 20 ++ .../io/udash/rest/StreamingRestTestApi.scala | 22 +- .../scala/io/udash/rest/raw/RawRestTest.scala | 226 ++++++++++++++++- .../io/udash/rest/raw/ServerImplApiTest.scala | 239 +++++++++++++++++- 9 files changed, 564 insertions(+), 26 deletions(-) diff --git a/rest/.jvm/src/test/resources/StreamingRestTestApi.json b/rest/.jvm/src/test/resources/StreamingRestTestApi.json index 51bfc9329..8ae1e2f59 100644 --- a/rest/.jvm/src/test/resources/StreamingRestTestApi.json +++ b/rest/.jvm/src/test/resources/StreamingRestTestApi.json @@ -24,6 +24,49 @@ } } }, + "/delayedStream": { + "get": { + "operationId": "delayedStream", + "parameters": [ + { + "name": "size", + "in": "query", + "required": true, + "explode": false, + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "delayMillis", + "in": "query", + "required": true, + "explode": false, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + } + } + } + } + } + } + }, "/errorStream": { "post": { "operationId": "errorStream", diff --git a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala index 6d2b9f6cf..03b99c17a 100644 --- a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala +++ b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala @@ -100,7 +100,7 @@ import scala.concurrent.duration.* Observable .fromIterator(Task.eval(input.readList().iterator(_.asInstanceOf[JsonStringInput].readRawJson()))) .map(JsonValue(_)) - }, + }.onErrorFallbackTo(Observable.raiseError(HttpErrorException.Streaming)), charset = charset, ) } diff --git a/rest/src/main/scala/io/udash/rest/companions.scala b/rest/src/main/scala/io/udash/rest/companions.scala index 65eda2893..bfea30d58 100644 --- a/rest/src/main/scala/io/udash/rest/companions.scala +++ b/rest/src/main/scala/io/udash/rest/companions.scala @@ -150,4 +150,7 @@ abstract class RestServerOpenApiImplCompanion[Implicits, Real](protected val imp final def asHandleRequest(real: Real): RawRest.HandleRequest = RawRest.asHandleRequest(real) + + final def asHandleRequestWithStreaming(real: Real): RawRest.HandleRequestWithStreaming = + RawRest.asHandleRequestWithStreaming(real) } diff --git a/rest/src/main/scala/io/udash/rest/raw/RestRequest.scala b/rest/src/main/scala/io/udash/rest/raw/RestRequest.scala index 9f690404d..860e6edb2 100644 --- a/rest/src/main/scala/io/udash/rest/raw/RestRequest.scala +++ b/rest/src/main/scala/io/udash/rest/raw/RestRequest.scala @@ -2,9 +2,9 @@ package io.udash package rest package raw -import com.avsystem.commons.meta._ +import com.avsystem.commons.meta.* import com.avsystem.commons.misc.{AbstractValueEnum, AbstractValueEnumCompanion, EnumCtx} -import com.avsystem.commons.rpc._ +import com.avsystem.commons.rpc.* import scala.util.control.NoStackTrace @@ -56,6 +56,7 @@ case class HttpErrorException(code: Int, payload: HttpBody = HttpBody.Empty, cau object HttpErrorException { def plain(code: Int, message: String, cause: Throwable = null): HttpErrorException = HttpErrorException(code, HttpBody.plain(message), cause) + val Streaming: HttpErrorException = plain(400, "HTTP stream failure") } final case class RestRequest(method: HttpMethod, parameters: RestParameters, body: HttpBody) { diff --git a/rest/src/test/scala/io/udash/rest/RestApiTest.scala b/rest/src/test/scala/io/udash/rest/RestApiTest.scala index ceb59b35b..5e631c429 100644 --- a/rest/src/test/scala/io/udash/rest/RestApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/RestApiTest.scala @@ -4,7 +4,7 @@ package rest import cats.implicits.catsSyntaxTuple2Semigroupal import com.avsystem.commons.* import com.avsystem.commons.misc.ScalaDurationExtensions.durationIntOps -import io.udash.rest.raw.RawRest +import io.udash.rest.raw.{HttpErrorException, RawRest} import io.udash.testing.AsyncUdashSharedTest import monix.eval.Task import monix.execution.Scheduler @@ -13,6 +13,7 @@ import org.scalactic.source.Position import org.scalatest.time.{Millis, Seconds, Span} import org.scalatest.{Assertion, BeforeAndAfterEach} +import scala.concurrent.TimeoutException import scala.concurrent.duration.FiniteDuration abstract class RestApiTest extends AsyncUdashSharedTest with BeforeAndAfterEach { @@ -28,7 +29,7 @@ abstract class RestApiTest extends AsyncUdashSharedTest with BeforeAndAfterEach override protected def beforeEach(): Unit = { super.beforeEach() - impl.resetCounter() + impl.resetCounter() // Reset non-streaming counter } final val serverHandle: RawRest.HandleRequest = @@ -168,8 +169,29 @@ trait StreamingRestApiTestScenarios extends RestApiTest { testStream(_.errorStream(immediate = true)) } - // TODO streaming - does not work on client side - "mid-stream error" ignore { + "mid-stream error" in { testStream(_.errorStream(immediate = false)) } + + "slow source stream" in { + testStream(_.delayedStream(size = 3, delayMillis = 100)) + } + + "client-side timeout on slow stream" in { + val streamTask = streamingProxy + .delayedStream(size = 10, delayMillis = 200) + .toListL + + val timeoutTask = streamTask.timeout(500.millis).materialize + + timeoutTask.runToFuture.map { result => + assert(result.isFailure, "Stream should have failed due to timeout") + result match { + case Failure(ex) => + assert(ex.isInstanceOf[TimeoutException], s"Expected TimeoutException, but got $ex") + succeed + case Success(_) => fail("Stream succeeded unexpectedly despite timeout") + } + } + } } diff --git a/rest/src/test/scala/io/udash/rest/SomeServerApiImpl.scala b/rest/src/test/scala/io/udash/rest/SomeServerApiImpl.scala index 48713ae7c..488442d5c 100644 --- a/rest/src/test/scala/io/udash/rest/SomeServerApiImpl.scala +++ b/rest/src/test/scala/io/udash/rest/SomeServerApiImpl.scala @@ -1,11 +1,31 @@ package io.udash.rest +import monix.reactive.Observable + import scala.concurrent.Future final class SomeServerApiImpl { @GET def thingy(param: Int): Future[String] = Future.successful((param - 1).toString) + @GET + def streamingNumbers(count: Int): Observable[Int] = + Observable.fromIterable(1 to count) + + @POST + def streamEcho(values: List[Int]): Observable[Int] = + Observable.fromIterable(values) + + @GET + def streamBinary(chunkSize: Int): Observable[Array[Byte]] = { + val content = "HelloWorld".getBytes + Observable.fromIterable(content.grouped(chunkSize).toSeq) + } + + @GET + def streamEmpty(): Observable[Array[Byte]] = + Observable.empty + val subapi = new SomeServerSubApiImpl } object SomeServerApiImpl extends DefaultRestServerApiImplCompanion[SomeServerApiImpl] diff --git a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala index 606eb2b0f..d46b6c6ce 100644 --- a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala @@ -2,10 +2,11 @@ package io.udash package rest import io.udash.rest.raw.HttpErrorException -import monix.execution.Scheduler +import monix.eval.Task import monix.reactive.Observable -import scala.concurrent.duration._ +import java.util.concurrent.atomic.AtomicBoolean +import scala.concurrent.duration.* trait StreamingRestTestApi { @GET def simpleStream(size: Int): Observable[Int] @@ -15,11 +16,11 @@ trait StreamingRestTestApi { @POST def binaryStream(): Observable[Array[Byte]] @POST def errorStream(@Query immediate: Boolean): Observable[RestEntity] + + @GET def delayedStream(@Query size: Int, @Query delayMillis: Long): Observable[Int] } object StreamingRestTestApi extends DefaultRestApiCompanion[StreamingRestTestApi] { - import Scheduler.Implicits.global - final class Impl extends StreamingRestTestApi { override def simpleStream(size: Int): Observable[Int] = @@ -40,7 +41,14 @@ object StreamingRestTestApi extends DefaultRestApiCompanion[StreamingRestTestApi else Observable.fromIterable(Range(0, 3)).map { i => if (i < 2) RestEntity(RestEntityId(i.toString), "first") - else throw HttpErrorException.plain(400, "later bad") - } + else throw HttpErrorException.Streaming + } + + + override def delayedStream(size: Int, delayMillis: Long): Observable[Int] = { + Observable.fromIterable(Range(0, size)) + .zip(Observable.intervalAtFixedRate(delayMillis.millis, delayMillis.millis)) + .map(_._1) + } } -} +} \ No newline at end of file diff --git a/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala b/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala index f120313e7..d8202ba8d 100644 --- a/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala +++ b/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala @@ -2,15 +2,20 @@ package io.udash package rest package raw -import com.avsystem.commons._ +import com.avsystem.commons.* import com.avsystem.commons.annotation.AnnotationAggregate -import com.avsystem.commons.serialization.{StringWrapperCompanion, transientDefault, whenAbsent} +import com.avsystem.commons.serialization.{transientDefault, whenAbsent} import io.udash.rest.util.WithHeaders import monix.eval.Task import monix.execution.Scheduler +import monix.reactive.Observable import org.scalactic.source.Position +import org.scalatest.Assertion import org.scalatest.concurrent.ScalaFutures import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import scala.concurrent.duration.* case class UserId(id: String) extends AnyVal { override def toString: String = id @@ -62,6 +67,13 @@ trait UserApi { def adjusted: Future[Unit] @CustomBody def binaryEcho(bytes: Array[Byte]): Future[Array[Byte]] + + @GET def streamNumbers(@Query("count") count: Int): Observable[Int] + @GET def streamStrings(@Query("count") count: Int, @Query("prefix") prefix: String): Observable[String] + @GET def streamUsers(@Query("count") count: Int): Observable[User] + @GET def streamEmpty: Observable[String] + @CustomBody def streamBinary(bytes: Array[Byte]): Observable[Array[Byte]] + @GET("streamError") def streamWithError(@Query("failAt") failAt: Int): Observable[String] } object UserApi extends DefaultRestApiCompanion[UserApi] @@ -74,8 +86,8 @@ trait RootApi { @POST @CustomBody def echoHeaders(headers: Map[String, String]): Future[WithHeaders[Unit]] } object RootApi extends DefaultRestApiCompanion[RootApi] - -class RawRestTest extends AnyFunSuite with ScalaFutures { +class RawRestTest extends AnyFunSuite with ScalaFutures with Matchers { + implicit override def patienceConfig: PatienceConfig = PatienceConfig(timeout = 5.seconds) implicit def scheduler: Scheduler = Scheduler.global def repr(body: HttpBody, inNewLine: Boolean = true): String = body match { @@ -106,6 +118,29 @@ class RawRestTest extends AnyFunSuite with ScalaFutures { s"<- ${resp.code}$headersRepr${repr(resp.body, hasHeaders)}".trim } + def repr(resp: StreamedRestResponse): Task[String] = { + val headersRepr = resp.headers.iterator + .map({ case (n, v) => s"$n: ${v.value}" }).mkStringOrEmpty("\n", "\n", "\n") + + val bodyReprTask: Task[String] = resp.body match { + case StreamedBody.Empty => Task.now("") + case StreamedBody.JsonList(elements, charset) => + elements.toListL.map { list => + s" application/json;charset=$charset\n[${list.map(_.value).mkString(",")}]" + } + case StreamedBody.RawBinary(content) => + content.toListL.map { list => + s" application/octet-stream\n${list.flatMap(_.iterator.map(b => f"$b%02X")).mkString}" + } + case StreamedBody.Single(body) => + Task.now(repr(body, inNewLine = false)) + } + + bodyReprTask.map { bodyRepr => + s"<- ${resp.code}$headersRepr$bodyRepr".trim + } + } + class RootApiImpl(id: Int, query: String) extends RootApi with UserApi { def self: UserApi = this def subApi(newId: Int, newQuery: String): UserApi = new RootApiImpl(newId, query + newQuery) @@ -124,9 +159,21 @@ class RawRestTest extends AnyFunSuite with ScalaFutures { def requireNonBlank(param: NonBlankString): Future[Unit] = Future.unit def echoHeaders(headers: Map[String, String]): Future[WithHeaders[Unit]] = Future.successful(WithHeaders((), headers.toList)) + def streamNumbers(count: Int): Observable[Int] = + Observable.fromIterable(1 to count) + def streamStrings(count: Int, prefix: String): Observable[String] = + Observable.fromIterable(1 to count).map(i => s"$prefix-$i") + def streamUsers(count: Int): Observable[User] = + Observable.fromIterable(1 to count).map(i => User(UserId(s"id-$i"), s"User $i")) + def streamEmpty: Observable[String] = Observable.empty + def streamBinary(bytes: Array[Byte]): Observable[Array[Byte]] = + Observable.fromIterable(bytes.grouped(2).toList) + def streamWithError(failAt: Int): Observable[String] = + Observable.fromIterable(1 to 10) + .map(i => if (i == failAt) throw HttpErrorException.plain(400, s"Error at $i") else s"item-$i") } - var trafficLog: String = _ + @volatile var trafficLog: String = _ val real: RootApi = new RootApiImpl(0, "") val serverHandle: RawRest.HandleRequest = request => @@ -136,24 +183,83 @@ class RawRestTest extends AnyFunSuite with ScalaFutures { } } + val serverHandleWithStreaming: RawRest.HandleRequestWithStreaming = request => + RawRest.asHandleRequestWithStreaming(real).apply(request).flatMap { response => + val logTask = response match { + case resp: RestResponse => + Task.eval { + trafficLog = s"${repr(request)}\n${repr(resp)}\n" + } + case streamResp: StreamedRestResponse => + repr(streamResp).map { responseRepr => + trafficLog = s"${repr(request)}\n$responseRepr\n" + } + } + logTask.as(response) + } + + val realProxy: RootApi = RawRest.fromHandleRequest[RootApi](serverHandle) + val realStreamingProxy: RootApi = RawRest.fromHandleRequestWithStreaming[RootApi](new RawRest.RestRequestHandler { + def handleRequest(request: RestRequest): Task[RestResponse] = + serverHandle(request) + + def handleRequestStream(request: RestRequest): Task[StreamedRestResponse] = + serverHandleWithStreaming(request).flatMap { + case stream: StreamedRestResponse => Task.now(stream) + case resp: RestResponse => + Task(StreamedRestResponse(resp.code, resp.headers, StreamedBody.fromHttpBody(resp.body), 1)) + } + }) + def testRestCall[T](call: RootApi => Future[T], expectedTraffic: String)(implicit pos: Position): Unit = { - assert(call(realProxy).wrapToTry.futureValue.map(mkDeep) == call(real).catchFailures.wrapToTry.futureValue.map(mkDeep)) - assert(trafficLog == expectedTraffic) + val realResult = call(real).catchFailures.wrapToTry.futureValue.map(mkDeep) + val proxyResult = call(realProxy).wrapToTry.futureValue.map(mkDeep) + proxyResult shouldBe realResult + trafficLog shouldBe expectedTraffic + } + + def testStreamingRestCall[T](call: RootApi => Observable[T], expectedTraffic: String)(implicit pos: Position): Unit = { + val realResultFuture = call(real).toListL.runToFuture.wrapToTry + val proxyResultFuture = call(realStreamingProxy).toListL.runToFuture.wrapToTry + + whenReady(realResultFuture) { realResult => + whenReady(proxyResultFuture) { proxyResult => + proxyResult shouldBe realResult + trafficLog shouldBe expectedTraffic + } + } } def mkDeep(value: Any): Any = value match { case arr: Array[_] => IArraySeq.empty[AnyRef] ++ arr.iterator.map(mkDeep) - case _ => value + case v => v } def assertRawExchange(request: RestRequest, response: RestResponse)(implicit pos: Position): Unit = { val future = serverHandle(request).runToFuture - assert(future.futureValue == response) + whenReady(future) { result => + result shouldBe response + } } - // TODO streaming add streaming tests + def assertRawStreamingExchange( + request: RestRequest, + assertResponse: StreamedRestResponse => Future[Assertion] + )(implicit pos: Position): Unit = { + val futureResponse: Future[AbstractRestResponse] = serverHandleWithStreaming(request).runToFuture + + val assertionFuture: Future[Assertion] = futureResponse.flatMap { + case stream: StreamedRestResponse => + assertResponse(stream) + case resp: RestResponse => + Future.successful(fail(s"Expected StreamedRestResponse but got RestResponse: $resp")) + } + whenReady(assertionFuture) { assertionResult => + assertionResult + } + } test("simple GET") { testRestCall(_.self.user(UserId("ID")), @@ -333,4 +439,102 @@ class RawRestTest extends AnyFunSuite with ScalaFutures { val response = RestResponse(400, IMapping.empty, HttpBody.plain("this stuff is blank")) assertRawExchange(request, response) } -} + + test("stream numbers") { + testStreamingRestCall(_.self.streamNumbers(5), + """-> GET /streamNumbers?count=5 + |<- 200 application/json;charset=utf-8 + |[1,2,3,4,5] + |""".stripMargin + ) + } + + test("stream strings with prefix") { + testStreamingRestCall(_.self.streamStrings(3, "test"), + """-> GET /streamStrings?count=3&prefix=test + |<- 200 application/json;charset=utf-8 + |["test-1","test-2","test-3"] + |""".stripMargin + ) + } + + test("stream users") { + testStreamingRestCall(_.self.streamUsers(2), + """-> GET /streamUsers?count=2 + |<- 200 application/json;charset=utf-8 + |[{"id":"id-1","name":"User 1"},{"id":"id-2","name":"User 2"}] + |""".stripMargin + ) + } + + test("stream empty") { + testStreamingRestCall(_.self.streamEmpty, + """-> GET /streamEmpty + |<- 200 application/json;charset=utf-8 + |[] + |""".stripMargin + ) + } + + test("stream binary") { + val inputBytes = Array[Byte](1, 2, 3, 4) + val expectedTraffic = + """-> POST /streamBinary application/octet-stream + |01020304 + |<- 200 application/octet-stream + |01020304 + |""".stripMargin + + val realResultFuture = real.self.streamBinary(inputBytes).toListL.runToFuture.map(_.map(_.toList)) + val proxyResultFuture = realStreamingProxy.self.streamBinary(inputBytes).toListL.runToFuture.map(_.map(_.toList)) + + whenReady(realResultFuture) { realResult => + whenReady(proxyResultFuture) { proxyResult => + proxyResult shouldBe realResult + trafficLog shouldBe expectedTraffic + } + } + } + + test("stream with error") { + val futureResult = realStreamingProxy.self.streamWithError(3).toListL.runToFuture.wrapToTry + + whenReady(futureResult) { resultTry => + val exception = intercept[HttpErrorException] { + resultTry.get + } + exception.code shouldBe 400 + exception.payload.textualContentOpt.get shouldBe "Error at 3" + } + } + + + test("streaming after prefix call") { + testStreamingRestCall(_.subApi(5, "test").streamNumbers(3), + """-> GET /subApi/5/streamNumbers?query=test&count=3 + |<- 200 application/json;charset=utf-8 + |[1,2,3] + |""".stripMargin + ) + } + + test("streaming response via raw exchange") { + val request = RestRequest( + HttpMethod.GET, + RestParameters(PlainValue.decodePath("streamNumbers"), query = Mapping(ISeq("count" -> PlainValue("4")))), + HttpBody.Empty + ) + + assertRawStreamingExchange(request, { response => + response.code shouldBe 200 + response.body match { + case StreamedBody.JsonList(elements, _) => + elements.toListL.runToFuture.map { elementList => + elementList.map(_.value) shouldBe List("1", "2", "3", "4") + } + case other => + Future.successful(fail(s"Expected JsonList body, got $other")) + } + }) + } +} \ No newline at end of file diff --git a/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala b/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala index 4bff878ec..e8c4367af 100644 --- a/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala @@ -4,6 +4,7 @@ import com.avsystem.commons.serialization.json.JsonStringOutput import io.udash.rest.SomeServerApiImpl import io.udash.rest.openapi.Info import monix.execution.Scheduler +import monix.reactive.Observable import org.scalactic.source.Position import org.scalatest.concurrent.ScalaFutures import org.scalatest.funsuite.AnyFunSuite @@ -13,13 +14,51 @@ class ServerImplApiTest extends AnyFunSuite with ScalaFutures { private val apiImpl = new SomeServerApiImpl private val serverHandle = SomeServerApiImpl.asHandleRequest(apiImpl) + private val serverHandleStreaming = SomeServerApiImpl.asHandleRequestWithStreaming(apiImpl) def assertRawExchange(request: RestRequest, response: RestResponse)(implicit pos: Position): Unit = { val future = serverHandle(request).runToFuture assert(future.futureValue == response) } - // TODO streaming add streaming tests + def assertStreamingExchange( + request: RestRequest, + expectedCode: Int, + expectedContentType: Option[String] = None, + verifyBody: Observable[_] => Boolean = _ => true + )(implicit pos: Position): Unit = { + val future = serverHandleStreaming(request).runToFuture + val response = future.futureValue + + assert(response.isInstanceOf[StreamedRestResponse], "Response should be a StreamedRestResponse") + + val streamedResp = response.asInstanceOf[StreamedRestResponse] + assert(streamedResp.code == expectedCode, s"Expected status code $expectedCode but got ${streamedResp.code}") + + expectedContentType.foreach { contentType => + val actualContentType = streamedResp.body match { + case nonEmpty: StreamedBody.NonEmpty => nonEmpty.contentType + case _ => "" + } + assert(actualContentType.startsWith(contentType), + s"Expected content type starting with $contentType but got $actualContentType") + } + + streamedResp.body match { + case StreamedBody.Empty => + assert(expectedContentType.isEmpty, "Expected empty body but content type was specified") + + case jsonList: StreamedBody.JsonList => + assert(verifyBody(jsonList.elements), "JSON body verification failed") + + case binary: StreamedBody.RawBinary => + assert(verifyBody(binary.content), "Binary body verification failed") + + case single: StreamedBody.Single => + // For single body, we don't need to verify the observable as it's just a wrapper + assert(true) + } + } test("simple GET call") { val params = RestParameters( @@ -53,6 +92,130 @@ class ServerImplApiTest extends AnyFunSuite with ScalaFutures { | "version": "0.1" | }, | "paths": { + | "/streamBinary": { + | "get": { + | "operationId": "streamBinary", + | "parameters": [ + | { + | "name": "chunkSize", + | "in": "query", + | "required": true, + | "explode": false, + | "schema": { + | "type": "integer", + | "format": "int32" + | } + | } + | ], + | "responses": { + | "200": { + | "description": "Success", + | "content": { + | "application/octet-stream": { + | "schema": { + | "type": "string", + | "format": "binary" + | } + | } + | } + | } + | } + | } + | }, + | "/streamEcho": { + | "post": { + | "operationId": "streamEcho", + | "requestBody": { + | "content": { + | "application/json": { + | "schema": { + | "type": "object", + | "properties": { + | "values": { + | "type": "array", + | "items": { + | "type": "integer", + | "format": "int32" + | } + | } + | }, + | "required": [ + | "values" + | ] + | } + | } + | }, + | "required": true + | }, + | "responses": { + | "200": { + | "description": "Success", + | "content": { + | "application/json": { + | "schema": { + | "type": "array", + | "items": { + | "type": "integer", + | "format": "int32" + | } + | } + | } + | } + | } + | } + | } + | }, + | "/streamEmpty": { + | "get": { + | "operationId": "streamEmpty", + | "responses": { + | "200": { + | "description": "Success", + | "content": { + | "application/octet-stream": { + | "schema": { + | "type": "string", + | "format": "binary" + | } + | } + | } + | } + | } + | } + | }, + | "/streamingNumbers": { + | "get": { + | "operationId": "streamingNumbers", + | "parameters": [ + | { + | "name": "count", + | "in": "query", + | "required": true, + | "explode": false, + | "schema": { + | "type": "integer", + | "format": "int32" + | } + | } + | ], + | "responses": { + | "200": { + | "description": "Success", + | "content": { + | "application/json": { + | "schema": { + | "type": "array", + | "items": { + | "type": "integer", + | "format": "int32" + | } + | } + | } + | } + | } + | } + | } + | }, | "/subapi/yeet": { | "post": { | "operationId": "subapi_yeet", @@ -121,4 +284,78 @@ class ServerImplApiTest extends AnyFunSuite with ScalaFutures { | "components": {} |}""".stripMargin) } + + test("streaming GET with JSON response") { + val params = RestParameters( + path = PlainValue.decodePath("/streamingNumbers"), + query = Mapping.create("count" -> PlainValue("5")) + ) + val request = RestRequest(HttpMethod.GET, params, HttpBody.Empty) + + assertStreamingExchange( + request = request, + expectedCode = 200, + expectedContentType = Some(HttpBody.JsonType), + verifyBody = obs => { + val valuesFuture = obs.asInstanceOf[Observable[JsonValue]] + .map(json => json.value.trim.toInt) + .toListL.runToFuture + valuesFuture.futureValue == List(1, 2, 3, 4, 5) + } + ) + } + + test("streaming POST with JSON body and streamed response") { + val params = RestParameters( + path = PlainValue.decodePath("/streamEcho"), + ) + val body = HttpBody.createJsonBody(Mapping.create("values" -> JsonValue("[1,2,3,4,5]"))) + val request = RestRequest(HttpMethod.POST, params, body) + + assertStreamingExchange( + request = request, + expectedCode = 200, + expectedContentType = Some(HttpBody.JsonType), + verifyBody = obs => { + val valuesFuture = obs.asInstanceOf[Observable[JsonValue]] + .map(json => json.value.trim.toInt) + .toListL.runToFuture + valuesFuture.futureValue == List(1, 2, 3, 4, 5) + } + ) + } + + test("streaming binary data") { + val params = RestParameters( + path = PlainValue.decodePath("/streamBinary"), + query = Mapping.create("chunkSize" -> PlainValue("3")) + ) + val request = RestRequest(HttpMethod.GET, params, HttpBody.Empty) + + assertStreamingExchange( + request = request, + expectedCode = 200, + expectedContentType = Some(HttpBody.OctetStreamType), + verifyBody = obs => { + val chunksFuture = obs.asInstanceOf[Observable[Array[Byte]]] + .map(bytes => new String(bytes)) + .toListL.runToFuture + + chunksFuture.futureValue.mkString("") == "HelloWorld" + } + ) + } + + test("empty streaming response") { + val params = RestParameters( + path = PlainValue.decodePath("/streamEmpty") + ) + val request = RestRequest(HttpMethod.GET, params, HttpBody.Empty) + + assertStreamingExchange( + request = request, + expectedCode = 200 + ) + } } + From be1bc6355f62c4819e0bbae897a7c55b057f8d9b Mon Sep 17 00:00:00 2001 From: Krzysztof Maliszewski Date: Sat, 5 Apr 2025 11:14:48 +0200 Subject: [PATCH 068/162] Add streaming support documentation for REST API --- guide/guide/.js/src/main/assets/pages/rest.md | 357 ++++++++++++------ 1 file changed, 242 insertions(+), 115 deletions(-) diff --git a/guide/guide/.js/src/main/assets/pages/rest.md b/guide/guide/.js/src/main/assets/pages/rest.md index f809010c8..21fce10c1 100644 --- a/guide/guide/.js/src/main/assets/pages/rest.md +++ b/guide/guide/.js/src/main/assets/pages/rest.md @@ -3,7 +3,7 @@ Udash framework contains an RPC based REST framework for defining REST services using plain Scala traits. It may be used for implementing both client and server side and works in both JVM and JS, as long as appropriate network layer is implemented. By default, Udash provides Java Servlet based server -implementation and [sttp](https://github.com/softwaremill/sttp) based client implementation +implementation and [sttp](https://github.com/softwaremill/sttp) based client implementation (which works in both JVM and JS). Udash REST is a module completely independent of other parts of Udash. @@ -18,30 +18,30 @@ into your dependencies (e.g. UI related modules). Udash REST: -* Provides automatic translation of **plain Scala traits** into REST endpoints - * Lets you cover your web endpoint with nice, typesafe, well organized, IDE-friendly language-level interface. - * Forms a type safety layer between the client and the server -* Gives you a set of annotations for adjusting how the translation into an HTTP endpoint happens. -* Statically validates your trait, emitting **detailed and readable compilation errors** in case anything is wrong. -* Uses typeclass-based, boilerplate free, pluggable and extensible serialization. You can easily integrate your +- Provides automatic translation of **plain Scala traits** into REST endpoints + - Lets you cover your web endpoint with nice, typesafe, well organized, IDE-friendly language-level interface. + - Forms a type safety layer between the client and the server +- Gives you a set of annotations for adjusting how the translation into an HTTP endpoint happens. +- Statically validates your trait, emitting **detailed and readable compilation errors** in case anything is wrong. +- Uses typeclass-based, boilerplate free, pluggable and extensible serialization. You can easily integrate your favorite serialization library into it. -* Uses pluggable and extensible effects for asynchronous IO. You can easily integrate your favorite async - IO effect with it, be it `Future`, Monix `Task`, one of the `IO` monad implementations, etc. Blocking API is +- Uses pluggable and extensible effects for asynchronous IO. You can easily integrate your favorite async + IO effect with it, be it `Future`, Monix `Task`, one of the `IO` monad implementations, etc. Blocking API is also possible. -* Is agnostic about being purely functional or not. You can use it with both programming styles. -* Automatically generates **OpenAPI** documents for your APIs. -* Has multiple ways of adjusting generated OpenAPI definition - * Provides a set of standard adjusting annotations, e.g. `@description` - * Lets you define your own adjusting annotations which may perform arbitrary modifications - * Gives you a nice, case class based representation of OpenAPI document which can be modified programmatically -* Uses pluggable network layer. You can easily integrate it with your favorite HTTP client and server. +- Is agnostic about being purely functional or not. You can use it with both programming styles. +- Automatically generates **OpenAPI** documents for your APIs. +- Has multiple ways of adjusting generated OpenAPI definition + - Provides a set of standard adjusting annotations, e.g. `@description` + - Lets you define your own adjusting annotations which may perform arbitrary modifications + - Gives you a nice, case class based representation of OpenAPI document which can be modified programmatically +- Uses pluggable network layer. You can easily integrate it with your favorite HTTP client and server. ## Quickstart example ### Project setup First, make sure appropriate dependencies are configured for your project. -Udash REST provides Servlet-based implementation for REST servers but a servlet must be run inside an HTTP server. +Udash REST provides Servlet-based implementation for REST servers but a servlet must be run inside an HTTP server. In this example we will use [Jetty](https://www.eclipse.org/jetty/) for that purpose. ```scala @@ -99,7 +99,7 @@ object ServerMain { def main(args: Array[String]): Unit = { // translate UserApiImpl into a Servlet val userApiServlet = RestServlet[UserApi](new UserApiImpl) - + // do all the Jetty related plumbing val server = new Server(9090) val handler = new ServletContextHandler @@ -127,7 +127,7 @@ object ClientMain { def main(args: Array[String]): Unit = { // allocate an STTP backend implicit val sttpBackend: SttpBackend[Future, Any] = SttpRestClient.defaultBackend() - + // obtain a "proxy" instance of UserApi val client: UserApi = SttpRestClient[UserApi]("http://localhost:9090/") @@ -154,6 +154,7 @@ object ClientMain { If we look at HTTP traffic created by the previous example, that's what we'll see: Request: + ``` POST http://localhost:9090/createUser HTTP/1.1 Accept-Encoding: gzip @@ -166,6 +167,7 @@ Content-Length: 32 ``` Response: + ``` HTTP/1.1 200 OK Date: Wed, 18 Jul 2018 11:43:08 GMT @@ -183,25 +185,25 @@ This approach is analogous to various well-established REST frameworks for other However, such frameworks are usually based on runtime reflection while in Scala it can be done using compile-time reflection through macros which offers several advantages: -* platform independence - REST traits are understood by both ScalaJVM and ScalaJS -* full type information - compile-time reflection is not limited by type erasure -* type safety - compile-time reflection can perform thorough validation of REST traits and +- platform independence - REST traits are understood by both ScalaJVM and ScalaJS +- full type information - compile-time reflection is not limited by type erasure +- type safety - compile-time reflection can perform thorough validation of REST traits and raise compilation errors in case anything is wrong -* pluggable typeclass based serialization - for serialization of REST parameters and results, +- pluggable typeclass based serialization - for serialization of REST parameters and results, typeclasses are used which also offers strong compile-time safety. If any of your parameters or method results cannot be serialized, a detailed compilation error will be raised. -* significantly better annotation processing +- significantly better annotation processing ### Companion objects In order for a trait to be understood as REST API, it must have a well defined companion object that contains appropriate implicits: -* in order to expose REST API on a server, implicit instances of `RawRest.AsRawRpc` and `RestMetadata` for API trait are required. -* in order to use REST API client, implicit instances of `RawRest.AsRealRpc` and `RestMetadata` for API trait are required. -* when API trait is used by both client and server, `RawRest.AsRawRpc` and `RawRest.AsRealRpc` may be provided by a single +- in order to expose REST API on a server, implicit instances of `RawRest.AsRawRpc` and `RestMetadata` for API trait are required. +- in order to use REST API client, implicit instances of `RawRest.AsRealRpc` and `RestMetadata` for API trait are required. +- when API trait is used by both client and server, `RawRest.AsRawRpc` and `RawRest.AsRealRpc` may be provided by a single combined instance of `RawRest.AsRawRealRpc` for API trait. -* additionally, if you want to [generate OpenAPI documents](#generating-openapi-30-specifications) then you need an instance of `OpenApiMetadata` +- additionally, if you want to [generate OpenAPI documents](#generating-openapi-30-specifications) then you need an instance of `OpenApiMetadata` Usually there is no need to declare these implicit instances manually because you can use one of the convenience base classes for REST API companion objects, e.g. @@ -217,7 +219,7 @@ object MyApi extends DefaultRestApiCompanion[MyApi] materialize all the necessary typeclass instances mentioned earlier. The "`Default`" in its name means that `DefaultRestImplicits` is used as a provider of serialization-related implicits. This effectively plugs [`GenCodec`](https://github.com/AVSystem/scala-commons/blob/master/docs/GenCodec.md) as the default serialization -library and `Future` as the default asynchronous effect for method results. +library and `Future` as the default asynchronous effect for method results. See [serialization](#serialization) for more details on customizing serialization. `DefaultRestApiCompanion` provides all the implicit instances necessary for both the client and server. @@ -230,10 +232,10 @@ generated code and make compilation faster. On less frequent occasions you might be unable to use one of the companion base classes. This is usually necessary when macro materialization requires some additional implicits or when your API trait takes type parameters. The recommended way of dealing with this situation is to design your own version of base companion class specialized -for your use case. For more details on how to do this, consult the Scaladoc of +for your use case. For more details on how to do this, consult the Scaladoc of [`MacroInstances`](https://github.com/AVSystem/scala-commons/blob/master/commons-core/src/main/scala/com/avsystem/commons/meta/MacroInstances.scala). -Ultimately, you can resort to declaring all the implicit instances manually (however, they will still be implemented +Ultimately, you can resort to declaring all the implicit instances manually (however, they will still be implemented with a macro). For example: ```scala @@ -274,7 +276,7 @@ object Address extends RestDataCompanion[Address] #### `RestDataWrapperCompanion` `RestDataWrapperCompanion` is a handy base companion class which you can use for data types which simply wrap -another type. It will establish a relation between the wrapping and wrapped types so that all REST-related implicits +another type. It will establish a relation between the wrapping and wrapped types so that all REST-related implicits for the wrapping type are automatically derived from corresponding implicits for the wrapped type. ```scala @@ -286,11 +288,11 @@ object UserId extends RestDataWrapperCompanion[String, UserId] REST framework relies on annotations for customization of REST API traits. All annotations are governed by the same [annotation processing](https://github.com/AVSystem/scala-commons/blob/master/docs/Annotations.md) rules -and extensions, implemented by the underlying macro engine from +and extensions, implemented by the underlying macro engine from [AVSystem Commons](https://github.com/AVSystem/scala-commons) library. To use annotations more effectively and with less boilerplate, it is highly recommended to be familiar with these rules. -The most important feature of annotation processing engine is an ability to create +The most important feature of annotation processing engine is an ability to create [`AnnotationAggregate`s](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/annotation/AnnotationAggregate.html). An annotation aggregate is a user-defined annotation which effectively applies a bunch of other annotations. This is a primary mechanism of code reuse in the area of annotations. It lets you significantly reduce annotation related boilerplate. @@ -300,26 +302,26 @@ related boilerplate. REST macro engine inspects an API trait and looks for all abstract methods. It then tries to translate every abstract method into an HTTP REST call. -* By default (if not annotated explicitly) each method is interpreted as HTTP `POST`. -* Method name is appended to the URL path. This can also be customized with annotations. -* Every parameter is interpreted as part of the body - by default all the body parameters will be +- By default (if not annotated explicitly) each method is interpreted as HTTP `POST`. +- Method name is appended to the URL path. This can also be customized with annotations. +- Every parameter is interpreted as part of the body - by default all the body parameters will be combined into a JSON object sent through HTTP body. If your method is annotated with [`@GET`](#get-methods) then it cannot send a body and method parameters are interpreted as query parameters rather than body fields. - You may also use other body formats by annotating your method as [`@FormBody`](#formbody) or + You may also use other body formats by annotating your method as [`@FormBody`](#formbody) or [`@CustomBody`](#custombody). -* Result type of each method is typically expected to be a `Future` wrapping some +- Result type of each method is typically expected to be a `Future` wrapping some arbitrary response type. This response type will be serialized into HTTP response which by default uses JSON for response body and creates a `200 OK` response with `application/json` content type. If response type is `Unit` (method result type is `Future[Unit]`) then a `204 No Content` response with empty body is created when serializing and body is ignored when deseriarlizing. -* Each method may also throw a `HttpErrorException` (or return failed `Future` with it). It will be +- Each method may also throw a `HttpErrorException` (or return failed `Future` with it). It will be automatically translated into appropriate HTTP error response with given status code and plaintext message. For details on how exactly serialization works and how to customize it, see [serialization](#serialization). -Note that if you don't want to use `Future`, this customization allows you to use other wrappers for method +Note that if you don't want to use `Future`, this customization allows you to use other wrappers for method result types. Through customized serialization it is also possible to signal HTTP errors without relying on -`HttpErrorException` or generally on throwing exceptions. This way you can customize the framework for more purely +`HttpErrorException` or generally on throwing exceptions. This way you can customize the framework for more purely functional programming style. ### Choosing the HTTP method @@ -415,7 +417,7 @@ as query parameters by default, so this annotation is necessary only for paramet `@Query` annotation also takes optional `name` parameter which may be specified to customize URL parameter name. If not specified, Scala parameter name is used. -Values of query parameters are serialized into `PlainValue` objects. +Values of query parameters are serialized into `PlainValue` objects. See [serialization](#path-query-header-and-cookie-serialization) for more details. #### Header parameters @@ -434,8 +436,8 @@ Scala parameter name is used. #### Body parameters -Every parameter of an API trait method (except for `@GET`) is interpreted as a field of a JSON object sent as -HTTP body. Just like for path, query, header and cookie parameters, there is a `@Body` annotation which requests this +Every parameter of an API trait method (except for `@GET`) is interpreted as a field of a JSON object sent as +HTTP body. Just like for path, query, header and cookie parameters, there is a `@Body` annotation which requests this explicitly. However, the only reason to use it explicitly is in order to customize the name of JSON field. Body parameters are serialized into `JsonValue` objects. @@ -462,7 +464,7 @@ object User extends RestDataCompanion[User] #### Optional parameters -Instead of `@Query`, `@Header`, `@Cookie` and `@Body`, you can also use `@OptQuery`, `@OptHeader`, `@OptCookie` +Instead of `@Query`, `@Header`, `@Cookie` and `@Body`, you can also use `@OptQuery`, `@OptHeader`, `@OptCookie` and `@OptBodyField` to make your parameters explicitly optional. The type of such a parameter must be wrapped into an `Option`, `Opt`, `OptArg` or similar option-like wrapper, i.e. @@ -490,8 +492,8 @@ possible lack of this parameter while the actual type of that parameter (that ne Prefix methods are methods that return other REST API traits. They are useful for: -* capturing common path or path/query/header/cookie parameters in a single prefix call -* splitting your REST API into multiple smaller traits in order to organize it better +- capturing common path or path/query/header/cookie parameters in a single prefix call +- splitting your REST API into multiple smaller traits in order to organize it better Just like HTTP API methods (`GET`, `POST`, etc.), prefix methods have their own annotation that can be used explicitly when you want your trait method to be treated as @@ -537,9 +539,9 @@ case classes used as parameter types or result types of REST methods. There are two ways to define default values: -* Scala-level default value +- Scala-level default value - You can simply use + You can simply use [language level default parameter value](https://docs.scala-lang.org/tour/default-parameter-values.html) for your REST method parameters and case class parameters. They will be picked up during macro materialization and used as fallback values for missing parameters during deserialization. However, Scala-level default values cannot @@ -547,15 +549,18 @@ There are two ways to define default values: by Scala compiler (obtaining such value requires an actual instance of API trait). Therefore, it's recommended to define default values using `@whenAbsent` annotation -* Using `@whenAbsent` annotation +- Using `@whenAbsent` annotation Instead of defining Scala-level default value, you can use `@whenAbsent` annotation: + ```scala @GET def fetchUsers(@whenAbsent(".*") namePattern: String): List[User] ``` + This brings two advantages: - * The default value is for deserialization _only_ and does not affect programmer API, which is often desired. - * Value from `@whenAbsent` will be picked up by macro materialization of + + - The default value is for deserialization _only_ and does not affect programmer API, which is often desired. + - Value from `@whenAbsent` will be picked up by macro materialization of [OpenAPI documents](#generating-openapi-30-specifications) and included as default value in OpenAPI [Schema Objects](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.2.md#schemaObject) @@ -584,7 +589,7 @@ every parameter value and every method result into appropriate raw values which be easily sent through network. Serialization in REST framework is typeclass based, which is a typical, functional and typesafe approach to serialization in Scala. -Examples of typeclass based serialization libraries include +Examples of typeclass based serialization libraries include [GenCodec](https://github.com/AVSystem/scala-commons/blob/master/docs/GenCodec.md) (which is the default serialization used by this REST framework), [circe](https://circe.github.io/circe/) (one of the most popular JSON libraries for Scala) or [µPickle](http://www.lihaoyi.com/upickle/). @@ -595,11 +600,11 @@ Any of these solutions can be plugged into REST framework. Depending on the context where a type is used in a REST API trait, it will be serialized to a different _raw value_: -* path/query/header parameters are serialized into `PlainValue` -* body parameters are serialized into `JsonValue` (by default), `PlainValue` (for [`@FormBody`](#formbody) methods) +- path/query/header parameters are serialized into `PlainValue` +- body parameters are serialized into `JsonValue` (by default), `PlainValue` (for [`@FormBody`](#formbody) methods) or directly into `HttpBody` (for [`@CustomBody`](#custombody) methods). -* Response types are serialized into `RestResponse` -* Prefix result types (other REST API traits) are "serialized" into an instance of `RawRest`. +- Response types are serialized into `RestResponse` +- Prefix result types (other REST API traits) are "serialized" into an instance of `RawRest`. When a macro needs to serialize a value of some type (let's call it `Real`) to one of these raw types listed above (let's call it `Raw`) then it looks for an implicit instance of `AsRaw[Raw, Real]`. @@ -608,16 +613,16 @@ Additionally, an implicit instance of `AsRawReal[Raw, Real]` can serve as both. These implicit instances may come from multiple sources: -* implicit scope of the `Raw` type (e.g. its companion object) -* implicit scope of the `Real` type (e.g. its companion object) -* implicits plugged by REST API trait companion +- implicit scope of the `Raw` type (e.g. its companion object) +- implicit scope of the `Real` type (e.g. its companion object) +- implicits plugged by REST API trait companion (e.g. `DefaultRestApiCompanion` plugs in `DefaultRestImplicits`) -* imports +- imports Of course, these implicits may also depend on other implicits which effectively means that you can use whatever typeclass-based serialization library you want. For example, you can define an instance of `AsRaw[JsonValue, Real]` which actually uses -`Encoder[Real]` from [circe](https://circe.github.io/circe/). +`Encoder[Real]` from [circe](https://circe.github.io/circe/). See [Customizing serialization](#customizing-serialization) for more details. ### Serialization implicits summary @@ -632,7 +637,7 @@ results on client side). ### Path, query, header and cookie serialization Path, query, header and cookie parameter values are serialized into `PlainValue` which is a simple `String` wrapper. -This means that the macro engine looks for an instance of `AsRaw[PlainValue, T]` and/or `AsReal[PlainValue, T]` for +This means that the macro engine looks for an instance of `AsRaw[PlainValue, T]` and/or `AsReal[PlainValue, T]` for every parameter of type `T` (`AsRaw` for the client, `AsReal` for the server). There are no "global" implicits defined for `PlainValue`. They must be either imported, defined by each @@ -651,9 +656,9 @@ URL-encoding is also applied to query and cookie parameter _names_, in both actu ### Body parameter serialization Body parameters are by default serialized into `JsonValue` which is also a simple wrapper class over `String`, -but is importantly distinct from `PlainValue` because it must always contain a valid JSON string. +but is importantly distinct from `PlainValue` because it must always contain a valid JSON string. This is required because JSON body parameters are ultimately composed into a single -JSON object sent as HTTP body. If a method is annotated with [`@FormBody`](#formbody), body parameters are +JSON object sent as HTTP body. If a method is annotated with [`@FormBody`](#formbody), body parameters are serialized into `PlainValue` and combined into an URL-encoded form. There are no "global" implicits defined for `JsonValue` - JSON serialization must be either imported, @@ -676,11 +681,11 @@ serializable as `HttpBody`. ### Result serialization Result type of every REST API method is wrapped into `Try` (in case the method throws an exception) -and translated into `Task[RestResponse]`. This means that the macro engine looks for an implicit instance of +and translated into `Task[RestResponse]`. This means that the macro engine looks for an implicit instance of `AsRaw[Task[RestResponse], Try[R]]` and `AsReal[Task[RestResponse], Try[R]]` for every HTTP method with result type `R`. -* `Task` is `monix.eval.Task` and represents a repeatable, cancelable, asynchronous computation. -* `RestResponse` itself is a simple class that aggregates HTTP status code, response headers and body. +- `Task` is `monix.eval.Task` and represents a repeatable, cancelable, asynchronous computation. +- `RestResponse` itself is a simple class that aggregates HTTP status code, response headers and body. `DefaultRestApiCompanion` and its friends introduce implicits which translate between `Task` and `Future`s. This effectively means that if your method returns `Future[R]` then it's enough if `R` is serializable as `RestResponse`. @@ -736,13 +741,13 @@ level of control that you need. **WARNING**: Remember that if you generate [OpenAPI documents](#generating-openapi-30-specifications) for your REST API then you must also provide custom instance of one of the [OpenAPI typeclasses](#openapi-implicits-summary) -so that OpenAPI document properly reflects your custom serialization format. +so that OpenAPI document properly reflects your custom serialization format. -* If you have custom serialization to `JsonValue` or `PlainValue` then you should define custom +- If you have custom serialization to `JsonValue` or `PlainValue` then you should define custom [`RestSchema`](#restschema-typeclass) instance -* If you have custom serialization to `HttpBody` then you should define custom +- If you have custom serialization to `HttpBody` then you should define custom [`RestMediaTypes`](#restmediatypes-typeclass) instance -* If you have custom serialization to `RestResponse` then you should define custom +- If you have custom serialization to `RestResponse` then you should define custom [`RestResponses`](#restresponses-typeclass) instance #### Providing serialization for third party type @@ -782,7 +787,7 @@ Instead, it introduces a mechanism through which serialization implicits are inj not bound to any specific serialization library. At the same time it provides a concise method to inject serialization implicits that does not require importing them explicitly. -An example usage of this mechanism is `DefaultRestApiCompanion` which injects +An example usage of this mechanism is `DefaultRestApiCompanion` which injects [`GenCodec`](https://github.com/AVSystem/scala-commons/blob/master/docs/GenCodec.md)-based serialization. @@ -809,10 +814,10 @@ object CirceRestImplicits extends CirceRestImplicits ``` Note that implicits are wrapped into `Fallback`. This is not strictly required, but it's recommended -because these implicits ultimately will have to be imported into *lexical scope* during macro materialization. -However, we don't want these implicits to have higher priority than implicits from the companion objects of some -concrete classes which need custom (*implicit scope*). Because of that, we wrap our implicits into -`Fallback` which keeps them visible but without elevated priority. `Fallback` is then "unwrapped" by appropriate +because these implicits ultimately will have to be imported into _lexical scope_ during macro materialization. +However, we don't want these implicits to have higher priority than implicits from the companion objects of some +concrete classes which need custom (_implicit scope_). Because of that, we wrap our implicits into +`Fallback` which keeps them visible but without elevated priority. `Fallback` is then "unwrapped" by appropriate implicits defined in `AsRaw` and `AsReal` companion objects. Now, in order to define a REST API trait that uses Circe-based serialization, you must appropriately @@ -845,7 +850,7 @@ REST API, then along from custom serialization you must provide customized insta #### Adjusting client-side `Scheduler` used for `Future`-based methods -`DefaultRestImplicits` contains a method that specifies the `monix.execution.Scheduler` +`DefaultRestImplicits` contains a method that specifies the `monix.execution.Scheduler` (extended version of `ExecutionContext`, usually wraps a thread pool) that is used for serialization and deserialization between `RestRequest`/`RestResponse` and representations of requests and responses native to the HTTP client being used. @@ -864,7 +869,7 @@ object MyFutureBasedRestApi extends RestApiCompanion[CustomizedRestImplicits, My #### Supporting async effects other than `Task` and `Future` -When using `DefaultRestApiCompanion` or one of its variations, every HTTP method in REST API trait must return +When using `DefaultRestApiCompanion` or one of its variations, every HTTP method in REST API trait must return its return wrapped into a Monix `Task` or `Future`. It is possible to use other asynchronous IO effects. In order to do that, you must provide some additional implicits which will make the macro engine @@ -883,30 +888,31 @@ REST framework gives you a certain amount of guarantees about backwards compatib Here's a list of changes that you may safely do to your REST API traits without breaking clients that still use the old version: -* Adding new REST methods, as long as paths are still the same and unambiguous. -* Renaming REST methods, as long as old `path` is configured for them explicitly (e.g. `@GET("oldname") def newname(...)`) -* Reordering parameters of your REST methods, except for `@Path` parameters which may be freely intermixed +- Adding new REST methods, as long as paths are still the same and unambiguous. +- Renaming REST methods, as long as old `path` is configured for them explicitly (e.g. `@GET("oldname") def newname(...)`) +- Reordering parameters of your REST methods, except for `@Path` parameters which may be freely intermixed with other parameters but they must retain the same order relative to each other. -* Splitting parameters into multiple parameter lists or making them `implicit`. -* Extracting common path fragments or parameters into [prefix methods](#prefix-methods). -* Renaming `@Path` parameters - their names are not used in REST requests +- Splitting parameters into multiple parameter lists or making them `implicit`. +- Extracting common path fragments or parameters into [prefix methods](#prefix-methods). +- Renaming `@Path` parameters - their names are not used in REST requests (they are used when generating OpenAPI though) -* Renaming non-`@Path` parameters, as long as the previous name is explicitly configured by +- Renaming non-`@Path` parameters, as long as the previous name is explicitly configured by `@Query`, `@Header`, `@Cookie` or `@Body` annotation. -* Removing non-`@Path` parameters - even if the client sends them, the server will just ignore them. -* Adding new non-`@Path` parameters, as long as default value is provided for them - either as +- Removing non-`@Path` parameters - even if the client sends them, the server will just ignore them. +- Adding new non-`@Path` parameters, as long as default value is provided for them - either as Scala-level default parameter value or by using `@whenAbsent` annotation. The server will simply use the default value if parameter is missing in incoming HTTP request. -* Changing parameter or result types or their serialization - as long as serialized formats of new and old type +- Changing parameter or result types or their serialization - as long as serialized formats of new and old type are compatible. This depends on on the serialization library you're using. If you're using `GenCodec`, consult [its documentation on retaining backwards compatibility](https://github.com/AVSystem/scala-commons/blob/master/docs/GenCodec.md#safely-introducing-changes-to-serialized-classes-retaining-backwards-compatibility). - + Conversely, changes that would break your API include: -* Renaming REST methods without explicitly configuring path -* Renaming non-`@Path` parameters which don't have explicit name configured -* Adding or removing `@Path` parameters -* Adding non-`@Path` parameters without giving them default value -* Changing order of `@Path` parameters + +- Renaming REST methods without explicitly configuring path +- Renaming non-`@Path` parameters which don't have explicit name configured +- Adding or removing `@Path` parameters +- Adding non-`@Path` parameters without giving them default value +- Changing order of `@Path` parameters ## Implementing backends @@ -932,7 +938,7 @@ easily sent through network. `RestResponse` is, similarly, a simple representation of HTTP response. `RestResponse` is made of HTTP status code and HTTP body (`HttpBody`, which also contains media type). -Monix `Task` is currently used as an "IO monad" implementation, i.e. a suspended, repeatable and cancellable +Monix `Task` is currently used as an "IO monad" implementation, i.e. a suspended, repeatable and cancellable asynchronous computation. In other words, `HandleRequest` is a function which translates a `RestRequest` into an unexecuted, asynchronous @@ -940,12 +946,12 @@ computation which yields a `RestResponse` when run. ### Implementing a server -An existing implementation of REST API trait can be easily turned into a `HandleRequest` +An existing implementation of REST API trait can be easily turned into a `HandleRequest` function using `RawRest.asHandleRequest`. Therefore, the only thing you need to do to expose your REST API trait as an actual web service it to turn `HandleRequest` function into a server. This is usually just a matter of translating native HTTP request into -a `RestRequest`, passing them to `HandleRequest` function and translating resulting `RestResponse` to native +a `RestRequest`, passing them to `HandleRequest` function and translating resulting `RestResponse` to native HTTP response. See [`RestServlet`](../rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala) @@ -963,6 +969,130 @@ to turn this native HTTP client into a `HandleRequest` function. See [`DefaultRestClient`](../rest/src/main/scala/io/udash/rest/DefaultRestClient.scala) for an example implementation. +## Streaming Support + +Udash REST provides built-in support for streaming responses, allowing efficient transfer of large data sets without loading everything into memory at once. + +### Defining Streaming APIs + +To define a streaming API endpoint, simply return `Observable[T]` as your method result type: + +```scala +import monix.reactive.Observable + +trait StreamingApi { + /** Returns a potentially large stream of data */ + def streamItems(filter: String): Observable[Item] + + /** Streams a file as binary data */ + def downloadFile(id: String): Observable[Array[Byte]] +} +object StreamingApi extends DefaultRestApiCompanion[StreamingApi] +``` + +The framework will automatically recognize `Observable` return types and handle them appropriately. + +### Implementing Streaming on the Server + +On the server side, implement your streaming methods by returning Monix Observables: + +```scala +import monix.reactive.Observable + +class StreamingApiImpl extends StreamingApi { + def streamItems(filter: String): Observable[Item] = + // Create an observable that emits items incrementally + Observable.fromIterator(Task( + database.queryItems(filter).iterator + )) + + def downloadFile(id: String): Observable[Array[Byte]] = + // Stream file contents in chunks + Observable.fromIterator(Task( + FileReader.readChunks(getFile(id)) + )) +} +``` + +### Consuming Streams on the Client + +Clients consume streaming responses using the same API definition: + +```scala +val client: StreamingApi = SttpRestClient[StreamingApi]("http://localhost:9090/") + +// Process items as they arrive +client.streamItems("product") + .foreach(item => println(s"Received: ${item.name}"))(monixScheduler) + +// Process a file as it downloads +client.downloadFile("report.pdf") + .bufferTumbling(1024) + .foreach(chunk => outputStream.write(chunk))(monixScheduler) +``` + +### How Streaming Works + +When a client makes a request to a streaming endpoint: + +1. The server does not specify a `Content-Length` header in the response +2. The client detects the streaming nature of the response by the missing `Content-Length` +3. Data is transferred incrementally in chunks as it becomes available +4. The client processes each chunk as it arrives + +This approach allows processing of potentially unlimited amounts of data with minimal memory footprint on both the client and server. + +### Streaming Types + +Udash REST supports two main streaming content types: + +1. **JSON Lists** - A stream of JSON objects sent as a JSON array `[{...}, {...}, ...]` +2. **Raw Binary** - A stream of binary data chunks, for files or other binary content + +### Implementation Details + +- Under the hood, streaming is implemented using Monix `Observable` +- Binary streams are transmitted as raw byte arrays +- JSON streams are automatically serialized/deserialized between JSON and your data types +- The server can control batch size to optimize network usage versus memory consumption + +### Handling Large Response Collections + +When dealing with large collections, streaming is preferable to loading everything in memory: + +```scala +// Without streaming - entire list is loaded in memory +def getAllItems(): Future[List[Item]] + +// With streaming - items are processed incrementally +def streamAllItems(): Observable[Item] +``` + +The streaming version allows processing data incrementally, which is crucial for very large datasets that might exceed available memory. + +## Error Handling with Streaming + +Streaming endpoints handle errors similarly to regular endpoints. When an error occurs during streaming: + +```scala +// Server side +def streamItems(filter: String): Observable[Item] = + if (filter.isEmpty) + Observable.raiseError(HttpErrorException.plain(400, "Filter cannot be empty")) + else + Observable.fromIterator(Task(database.queryItems(filter).iterator)) + +// Client side +client.streamItems("") + .onErrorHandle { error => + println(s"Streaming error: $error") + Item.default // Provide fallback value + } + .foreach(processItem)(monixScheduler) +``` + +This allows graceful handling of errors that might occur during streaming operations. + ## Generating OpenAPI 3.0 specifications [OpenAPI](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.2.md) is an open standard for describing @@ -1063,10 +1193,7 @@ object User extends RestDataCompanion[User] // gives GenCodec + RestStructure + "format": "int32" } }, - "required": [ - "id", - "birthYear" - ] + "required": ["id", "birthYear"] } ``` @@ -1148,10 +1275,10 @@ rather than inlined schema. `RestMediaType` is an auxiliary typeclass which serves as a basis for `RestResponses` and `RestRequestBody` typeclasses. It captures all the possible media types which may be used in a request or response body for given Scala type. -Media types are represented using OpenAPI +Media types are represented using OpenAPI [Media Type Object](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.2.md#mediaTypeObject). -By default, `RestMediaTypes` instance is derived from `RestSchema` instance and `application/json` is assumed as +By default, `RestMediaTypes` instance is derived from `RestSchema` instance and `application/json` is assumed as the only available media type. You **should** define `RestMediaTypes` manually for every type which has custom serialization to `HttpBody` defined @@ -1170,16 +1297,16 @@ By default, if no specific `RestResponses` instance is provided, it is created b The resulting [Responses](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.2.md#responsesObject) will contain exactly one [Response](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.2.md#responseObject) -for HTTP status code `200 OK` with +for HTTP status code `200 OK` with [Media Types](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.2.md#mediaTypeObject) -inferred from `RestMediaTypes` instance. Also note that `RestMediaTypes` itself is by default derived from +inferred from `RestMediaTypes` instance. Also note that `RestMediaTypes` itself is by default derived from `RestSchema` You **should** define `RestResponses` manually for every type which has custom serialization -to `RestResponse` defined (`AsRaw/AsReal[RestResponse, T]`). In general, you may want to define -it manually every time you want to describe responses for status codes other than `200 OK`. +to `RestResponse` defined (`AsRaw/AsReal[RestResponse, T]`). In general, you may want to define +it manually every time you want to describe responses for status codes other than `200 OK`. -Also remember that `Responses` object can be adjusted locally, for each method, using annotations - +Also remember that `Responses` object can be adjusted locally, for each method, using annotations - see [Adjusting operations](#adjusting-operations). ### `RestRequestBody` typeclass @@ -1218,9 +1345,9 @@ Annotations extending `SchemaAdjuster` can arbitrarily transform a [Schema Object](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.2.md#schemaObject) and can be applied on: -* data types with macro-generated `RestSchema` -* case class fields of data types with macro generated `RestSchema` -* `@Body` parameters of REST methods +- data types with macro-generated `RestSchema` +- case class fields of data types with macro generated `RestSchema` +- `@Body` parameters of REST methods Schema adjusters do **NOT** work on path/header/query/cookie parameters and REST methods themselves. Instead use [parameter adjusters](#adjusting-parameters) and @@ -1263,5 +1390,5 @@ it will apply to all Path Item Objects associated with result of this prefix met ### Limitations -* Current representation of OpenAPI document does not support -[specification extensions](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.2.md#specificationExtensions). +- Current representation of OpenAPI document does not support + [specification extensions](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.2.md#specificationExtensions). From ccfc1006d41dd10459160f808649e965bbcb491f Mon Sep 17 00:00:00 2001 From: Krzysztof Maliszewski Date: Sat, 5 Apr 2025 15:40:34 +0200 Subject: [PATCH 069/162] Refactor RawRestTest to handle sjs limitations --- .../scala/io/udash/rest/raw/RawRestTest.scala | 48 +++++-------------- 1 file changed, 13 insertions(+), 35 deletions(-) diff --git a/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala b/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala index d8202ba8d..901ec67f4 100644 --- a/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala +++ b/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala @@ -5,18 +5,16 @@ package raw import com.avsystem.commons.* import com.avsystem.commons.annotation.AnnotationAggregate import com.avsystem.commons.serialization.{transientDefault, whenAbsent} +import io.udash.rest.raw.StreamedBody.castOrFail import io.udash.rest.util.WithHeaders import monix.eval.Task import monix.execution.Scheduler import monix.reactive.Observable import org.scalactic.source.Position -import org.scalatest.Assertion import org.scalatest.concurrent.ScalaFutures import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -import scala.concurrent.duration.* - case class UserId(id: String) extends AnyVal { override def toString: String = id } @@ -87,7 +85,6 @@ trait RootApi { } object RootApi extends DefaultRestApiCompanion[RootApi] class RawRestTest extends AnyFunSuite with ScalaFutures with Matchers { - implicit override def patienceConfig: PatienceConfig = PatienceConfig(timeout = 5.seconds) implicit def scheduler: Scheduler = Scheduler.global def repr(body: HttpBody, inNewLine: Boolean = true): String = body match { @@ -244,23 +241,6 @@ class RawRestTest extends AnyFunSuite with ScalaFutures with Matchers { } } - def assertRawStreamingExchange( - request: RestRequest, - assertResponse: StreamedRestResponse => Future[Assertion] - )(implicit pos: Position): Unit = { - val futureResponse: Future[AbstractRestResponse] = serverHandleWithStreaming(request).runToFuture - - val assertionFuture: Future[Assertion] = futureResponse.flatMap { - case stream: StreamedRestResponse => - assertResponse(stream) - case resp: RestResponse => - Future.successful(fail(s"Expected StreamedRestResponse but got RestResponse: $resp")) - } - whenReady(assertionFuture) { assertionResult => - assertionResult - } - } - test("simple GET") { testRestCall(_.self.user(UserId("ID")), """-> GET /user?userId=ID @@ -485,12 +465,12 @@ class RawRestTest extends AnyFunSuite with ScalaFutures with Matchers { |01020304 |""".stripMargin - val realResultFuture = real.self.streamBinary(inputBytes).toListL.runToFuture.map(_.map(_.toList)) - val proxyResultFuture = realStreamingProxy.self.streamBinary(inputBytes).toListL.runToFuture.map(_.map(_.toList)) + val realResultFuture = real.self.streamBinary(inputBytes).toListL.runToFuture + val proxyResultFuture = realStreamingProxy.self.streamBinary(inputBytes).toListL.runToFuture whenReady(realResultFuture) { realResult => whenReady(proxyResultFuture) { proxyResult => - proxyResult shouldBe realResult + proxyResult.map(_.toList) shouldBe realResult.map(_.toList) trafficLog shouldBe expectedTraffic } } @@ -524,17 +504,15 @@ class RawRestTest extends AnyFunSuite with ScalaFutures with Matchers { RestParameters(PlainValue.decodePath("streamNumbers"), query = Mapping(ISeq("count" -> PlainValue("4")))), HttpBody.Empty ) - - assertRawStreamingExchange(request, { response => - response.code shouldBe 200 - response.body match { - case StreamedBody.JsonList(elements, _) => - elements.toListL.runToFuture.map { elementList => - elementList.map(_.value) shouldBe List("1", "2", "3", "4") - } - case other => - Future.successful(fail(s"Expected JsonList body, got $other")) + whenReady(serverHandleWithStreaming(request).runToFuture) { + case StreamedRestResponse(code, headers, body, batchSize) => { + code shouldBe 200 + val elements = castOrFail[StreamedBody.JsonList](body).elements + whenReady(elements.toListL.runToFuture) { e => + e shouldBe List(JsonValue("1"), JsonValue("2"), JsonValue("3"), JsonValue("4")) + } } - }) + case _ => fail() + } } } \ No newline at end of file From ca1d4534f193022caaebb7136d37ef480c178db2 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 8 Apr 2025 23:47:58 +0000 Subject: [PATCH 070/162] Update selenium-java to 4.31.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index b5c7c5266..b3c9e398b 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -42,7 +42,7 @@ object Dependencies { val bootstrap4DatepickerVersion = "5.39.0" val momentJsVersion = "2.30.1" - val seleniumVersion = "4.30.0" + val seleniumVersion = "4.31.0" val webDriverManagerVersion = "6.0.0" val scalaJsBenchmarkVersion = "0.10.0" From 77f94ca198daa74947984bb2dbde1965b1b6185c Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 8 Apr 2025 23:48:01 +0000 Subject: [PATCH 071/162] Update jetty-client to 12.0.19 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index b5c7c5266..dbf45c2cb 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -28,7 +28,7 @@ object Dependencies { val scalaLoggingVersion = "3.9.5" - val jettyVersion = "12.0.18" + val jettyVersion = "12.0.19" val typesafeConfigVersion = "1.4.3" val flexmarkVersion = "0.64.8" val logbackVersion = "1.3.15" From 77fe74576e829c275ba20ab0c4b1aa012dd4215b Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Wed, 9 Apr 2025 15:52:58 +0200 Subject: [PATCH 072/162] Customizable response batch size --- .../scala/io/udash/rest/RestServlet.scala | 23 ++++++++++++++----- .../io/udash/rest/jetty/JettyRestClient.scala | 13 ++++++----- .../scala/io/udash/rest/annotations.scala | 12 +++++++--- .../scala/io/udash/rest/raw/RawRest.scala | 8 +++---- .../io/udash/rest/raw/RestResponse.scala | 10 ++++---- .../io/udash/rest/raw/StreamedBody.scala | 1 - .../scala/io/udash/rest/raw/RawRestTest.scala | 6 ++--- 7 files changed, 45 insertions(+), 28 deletions(-) diff --git a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala index ddad45eb3..e08c821b8 100644 --- a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala +++ b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala @@ -22,6 +22,7 @@ object RestServlet { final val DefaultHandleTimeout = 30.seconds final val DefaultMaxPayloadSize = 16 * 1024 * 1024L // 16MB final val CookieHeader = "Cookie" + final val DefaultStreamingBatchSize = 1 private final val BufferSize = 8192 /** @@ -31,21 +32,29 @@ object RestServlet { * @param handleTimeout maximum time the servlet will wait for results returned by REST API implementation * @param maxPayloadSize maximum acceptable incoming payload size, in bytes; * if exceeded, `413 Payload Too Large` response will be sent back + * @param defaultStreamingBatchSize default batch size for [[StreamedRestResponse]] */ @explicitGenerics def apply[RestApi: RawRest.AsRawRpc : RestMetadata]( apiImpl: RestApi, handleTimeout: FiniteDuration = DefaultHandleTimeout, maxPayloadSize: Long = DefaultMaxPayloadSize, + defaultStreamingBatchSize: Int = DefaultStreamingBatchSize, )(implicit scheduler: Scheduler ): RestServlet = - new RestServlet(RawRest.asHandleRequestWithStreaming[RestApi](apiImpl), handleTimeout, maxPayloadSize) + new RestServlet( + handleRequest = RawRest.asHandleRequestWithStreaming[RestApi](apiImpl), + handleTimeout = handleTimeout, + maxPayloadSize = maxPayloadSize, + defaultStreamingBatchSize = defaultStreamingBatchSize, + ) } class RestServlet( handleRequest: RawRest.HandleRequestWithStreaming, handleTimeout: FiniteDuration = DefaultHandleTimeout, maxPayloadSize: Long = DefaultMaxPayloadSize, + defaultStreamingBatchSize: Int = DefaultStreamingBatchSize, )(implicit scheduler: Scheduler ) extends HttpServlet with LazyLogging { @@ -120,14 +129,16 @@ class RestServlet( Task.eval(writeNonEmptyBody(response, single.body)) case binary: StreamedBody.RawBinary => response.setContentType(binary.contentType) - binary.content.bufferTumbling(stream.batchSize).consumeWith(Consumer.foreach { batch => - batch.foreach(e => response.getOutputStream.write(e)) - response.getOutputStream.flush() - }) + binary.content + .bufferTumbling(stream.customBatchSize.getOrElse(defaultStreamingBatchSize)) + .consumeWith(Consumer.foreach { batch => + batch.foreach(e => response.getOutputStream.write(e)) + response.getOutputStream.flush() + }) case jsonList: StreamedBody.JsonList => response.setContentType(jsonList.contentType) jsonList.elements - .bufferTumbling(stream.batchSize) + .bufferTumbling(stream.customBatchSize.getOrElse(defaultStreamingBatchSize)) .switchIfEmpty(Observable(Seq.empty)) .zipWithIndex .consumeWith(Consumer.foreach { case (batch, idx) => diff --git a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala index 03b99c17a..631039d3d 100644 --- a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala +++ b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala @@ -9,8 +9,8 @@ import io.udash.rest.util.Utils import io.udash.utils.URLEncoder import monix.eval.Task import monix.execution.{Callback, Scheduler} +import monix.reactive.Observable import monix.reactive.OverflowStrategy.Unbounded -import monix.reactive.{MulticastStrategy, Observable} import monix.reactive.subjects.{ConcurrentSubject, PublishToOneSubject} import org.eclipse.jetty.client.* import org.eclipse.jetty.http.{HttpCookie, HttpHeader, MimeTypes} @@ -31,7 +31,8 @@ import scala.concurrent.duration.* * @param client The configured Jetty `HttpClient` instance. * @param defaultMaxResponseLength Default maximum size (in bytes) for buffering non-streamed responses. * @param defaultTimeout Default timeout for requests. - */final class JettyRestClient( + */ +final class JettyRestClient( client: HttpClient, defaultMaxResponseLength: Int = JettyRestClient.DefaultMaxResponseLength, defaultTimeout: Duration = JettyRestClient.DefaultTimeout, @@ -115,7 +116,7 @@ import scala.concurrent.duration.* code = response.getStatus, headers = parseHeaders(response), body = body, - batchSize = 1, + customBatchSize = Opt.Empty, ) callback(Success(restResponse)) } @@ -149,7 +150,7 @@ import scala.concurrent.duration.* code = httpResp.getStatus, headers = parseHeaders(httpResp), body = StreamedBody.fromHttpBody(parseHttpBody(httpResp, this)), - batchSize = 1, + customBatchSize = Opt.Empty, ) callback(Success(restResponse)) } else { @@ -165,8 +166,8 @@ import scala.concurrent.duration.* } /** - * Creates a `RawRest.HandleRequest` which handles standard REST requests by buffering the entire response. - * This does *not* support streaming responses. + * Creates a [[RawRest.HandleRequest]] which handles standard REST requests by buffering the entire response. + * This does not support streaming responses. * * @param baseUrl The base URL for the REST service. * @param customMaxResponseLength Optional override for the maximum response length. diff --git a/rest/src/main/scala/io/udash/rest/annotations.scala b/rest/src/main/scala/io/udash/rest/annotations.scala index f2ee0a389..a19ce1a05 100644 --- a/rest/src/main/scala/io/udash/rest/annotations.scala +++ b/rest/src/main/scala/io/udash/rest/annotations.scala @@ -1,11 +1,12 @@ package io.udash package rest +import com.avsystem.commons.Opt import com.avsystem.commons.annotation.{AnnotationAggregate, defaultsToName} import com.avsystem.commons.meta.RealSymAnnotation -import com.avsystem.commons.rpc._ +import com.avsystem.commons.rpc.* import com.avsystem.commons.serialization.optionalParam -import io.udash.rest.raw._ +import io.udash.rest.raw.* import scala.annotation.StaticAnnotation @@ -362,6 +363,11 @@ class addRequestHeader(name: String, value: String) extends RequestAdjuster { * HTTP header to all outgoing responses generated for invocations of that method on the server side. */ class addResponseHeader(name: String, value: String) extends ResponseAdjuster with StreamedResponseAdjuster { - def adjustResponse(response: RestResponse): RestResponse = response.header(name, value) + override def adjustResponse(response: RestResponse): RestResponse = response.header(name, value) override def adjustResponse(response: StreamedRestResponse): StreamedRestResponse = response.header(name, value) } + +class streamingResponseBatchSize(size: Int) extends StreamedResponseAdjuster { + override def adjustResponse(response: StreamedRestResponse): StreamedRestResponse = + response.copy(customBatchSize = Opt.some(size)) +} diff --git a/rest/src/main/scala/io/udash/rest/raw/RawRest.scala b/rest/src/main/scala/io/udash/rest/raw/RawRest.scala index 776809b26..2f70265b5 100644 --- a/rest/src/main/scala/io/udash/rest/raw/RawRest.scala +++ b/rest/src/main/scala/io/udash/rest/raw/RawRest.scala @@ -142,9 +142,9 @@ trait RawRest { RawRest.resolveAndHandle(metadata)(handleResolvedWithStreaming) /** - * Handles a resolved REST call and returns a standard RestResponse. + * Handles a resolved REST call and returns a standard [[RestResponse]]. * This method is maintained for backward compatibility with non-streaming clients. - * It delegates to handleResolvedWithStreaming and converts any streaming response to a standard response. + * It delegates to [[handleResolvedWithStreaming]] and converts any streaming response to a standard response. */ def handleResolved(request: RestRequest, resolved: ResolvedCall): Task[RestResponse] = StreamedRestResponse.fallbackToRestResponse(handleResolvedWithStreaming(request, resolved)) @@ -254,7 +254,7 @@ object RawRest extends RawRpcCompanion[RawRest] { @implicitNotFound(InvalidTraitMessage) implicit def rawRestAsRawNotFound[T]: ImplicitNotFound[AsRaw[RawRest, T]] = ImplicitNotFound() - // client side + // client side without response streaming support def fromHandleRequest[Real: AsRealRpc : RestMetadata](handle: HandleRequest): Real = RawRest.asReal(new DefaultRawRest(Nil, RestMetadata[Real], RestParameters.Empty, new RawRest.RestRequestHandler { override def handleRequest(request: RestRequest): Task[RestResponse] = handle(request) @@ -266,7 +266,7 @@ object RawRest extends RawRpcCompanion[RawRest] { def fromHandleRequestWithStreaming[Real: AsRealRpc : RestMetadata](handleRequest: RawRest.RestRequestHandler): Real = RawRest.asReal(new DefaultRawRest(Nil, RestMetadata[Real], RestParameters.Empty, handleRequest)) - // server side + // server side without response streaming support def asHandleRequest[Real: AsRawRpc : RestMetadata](real: Real): HandleRequest = RawRest.asRaw(real).asHandleRequest(RestMetadata[Real]) diff --git a/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala b/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala index f0709d59e..578c277bd 100644 --- a/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala +++ b/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala @@ -113,13 +113,13 @@ trait RestResponseLowPrio { this: RestResponse.type => /** * Streaming REST response containing a status code, headers, and a streamed body. - * Unlike standard RestResponse, the body content can be delivered incrementally through a reactive stream. + * Unlike standard [[RestResponse]], the body content can be delivered incrementally through a reactive stream. */ final case class StreamedRestResponse( code: Int, headers: IMapping[PlainValue], body: StreamedBody, - batchSize: Int, + customBatchSize: Opt[Int] = Opt.Empty, ) extends AbstractRestResponse { def header(name: String, value: String): StreamedRestResponse = @@ -132,7 +132,7 @@ final case class StreamedRestResponse( object StreamedRestResponse extends StreamedRestResponseLowPrio { /** - * Converts a StreamedRestResponse to a standard RestResponse by materializing streamed content. + * Converts a [[StreamedRestResponse]] to a standard [[RestResponse]] by materializing streamed content. * This is useful for compatibility with APIs that don't support streaming. */ def fallbackToRestResponse(response: StreamedRestResponse): Task[RestResponse] = { @@ -158,7 +158,7 @@ object StreamedRestResponse extends StreamedRestResponseLowPrio { } /** - * Converts any AbstractRestResponse to a standard RestResponse by materializing streamed content if necessary. + * Converts any [[AbstractRestResponse]] to a standard [[RestResponse]] by materializing streamed content if necessary. * This is useful for compatibility with APIs that don't support streaming. */ def fallbackToRestResponse(response: Task[AbstractRestResponse]): Task[RestResponse] = @@ -168,7 +168,7 @@ object StreamedRestResponse extends StreamedRestResponseLowPrio { } def fromHttpError(error: HttpErrorException): StreamedRestResponse = - StreamedRestResponse(error.code, IMapping.empty, StreamedBody.fromHttpBody(error.payload), 1) + StreamedRestResponse(error.code, IMapping.empty, StreamedBody.fromHttpBody(error.payload)) class LazyOps(private val resp: () => StreamedRestResponse) extends AnyVal { def recoverHttpError: StreamedRestResponse = try resp() catch { diff --git a/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala b/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala index e39193815..6a6d5120a 100644 --- a/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala +++ b/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala @@ -21,7 +21,6 @@ sealed trait StreamedBody { code = defaultStatus, headers = IMapping.empty, body = this, - batchSize = 1, ) } object StreamedBody extends StreamedBodyLowPrio { diff --git a/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala b/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala index 901ec67f4..97047e624 100644 --- a/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala +++ b/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala @@ -84,6 +84,7 @@ trait RootApi { @POST @CustomBody def echoHeaders(headers: Map[String, String]): Future[WithHeaders[Unit]] } object RootApi extends DefaultRestApiCompanion[RootApi] + class RawRestTest extends AnyFunSuite with ScalaFutures with Matchers { implicit def scheduler: Scheduler = Scheduler.global @@ -206,7 +207,7 @@ class RawRestTest extends AnyFunSuite with ScalaFutures with Matchers { serverHandleWithStreaming(request).flatMap { case stream: StreamedRestResponse => Task.now(stream) case resp: RestResponse => - Task(StreamedRestResponse(resp.code, resp.headers, StreamedBody.fromHttpBody(resp.body), 1)) + Task(StreamedRestResponse(resp.code, resp.headers, StreamedBody.fromHttpBody(resp.body))) } }) @@ -505,13 +506,12 @@ class RawRestTest extends AnyFunSuite with ScalaFutures with Matchers { HttpBody.Empty ) whenReady(serverHandleWithStreaming(request).runToFuture) { - case StreamedRestResponse(code, headers, body, batchSize) => { + case StreamedRestResponse(code, headers, body, batchSize) => code shouldBe 200 val elements = castOrFail[StreamedBody.JsonList](body).elements whenReady(elements.toListL.runToFuture) { e => e shouldBe List(JsonValue("1"), JsonValue("2"), JsonValue("3"), JsonValue("4")) } - } case _ => fail() } } From afbaf13b73b37cac634100497138c1fd6ef140e9 Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Thu, 10 Apr 2025 10:32:41 +0200 Subject: [PATCH 073/162] Update webdrivermanager to 6.0.1 (#1340) Co-authored-by: Dawid Dworak --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index cc6327cd4..74b285c30 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -43,7 +43,7 @@ object Dependencies { val momentJsVersion = "2.30.1" val seleniumVersion = "4.31.0" - val webDriverManagerVersion = "6.0.0" + val webDriverManagerVersion = "6.0.1" val scalaJsBenchmarkVersion = "0.10.0" val compilerPlugins = Def.setting(Seq( From 9c5edd0a52632a8e891c64ef5906e2b5e2456375 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Thu, 10 Apr 2025 19:53:00 +0000 Subject: [PATCH 074/162] Update monix to 3.11.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 74b285c30..dfcefb172 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -24,7 +24,7 @@ object Dependencies { val circeDerivationVersion = "0.13.0-M5" // Tests only val monixVersion = "3.4.1" // udash-rest only - val sttpVersion = "3.10.3" + val sttpVersion = "3.11.0" val scalaLoggingVersion = "3.9.5" From ce3711e63cd206f94f4ba27cd9f9657d8a96bd7d Mon Sep 17 00:00:00 2001 From: Krzysztof Maliszewski Date: Fri, 11 Apr 2025 11:21:12 +0200 Subject: [PATCH 075/162] address mr comments --- .../io/udash/rest/jetty/JettyRestClient.scala | 45 +++++++++------- .../main/scala/io/udash/rest/companions.scala | 17 +++++- .../main/scala/io/udash/rest/implicits.scala | 4 +- .../io/udash/rest/CompilationErrorsTest.scala | 54 +++++++++++++++++-- .../io/udash/rest/StreamingRestTestApi.scala | 5 +- .../io/udash/rest/raw/ServerImplApiTest.scala | 10 ++++ 6 files changed, 106 insertions(+), 29 deletions(-) diff --git a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala index 631039d3d..4ea5b2b65 100644 --- a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala +++ b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala @@ -4,11 +4,13 @@ package rest.jetty import com.avsystem.commons.* import com.avsystem.commons.annotation.explicitGenerics import com.avsystem.commons.serialization.json.{JsonReader, JsonStringInput} +import io.udash.rest.jetty.JettyRestClient.unsupportedContentTypeError import io.udash.rest.raw.* +import io.udash.rest.raw.HttpErrorException.plain import io.udash.rest.util.Utils import io.udash.utils.URLEncoder import monix.eval.Task -import monix.execution.{Callback, Scheduler} +import monix.execution.{Ack, Callback, Scheduler} import monix.reactive.Observable import monix.reactive.OverflowStrategy.Unbounded import monix.reactive.subjects.{ConcurrentSubject, PublishToOneSubject} @@ -28,9 +30,9 @@ import scala.concurrent.duration.* * response body in memory. This client activates streaming mode *only* when the server's * response headers *do not* include a `Content-Length`. * - * @param client The configured Jetty `HttpClient` instance. + * @param client The configured Jetty `HttpClient` instance. * @param defaultMaxResponseLength Default maximum size (in bytes) for buffering non-streamed responses. - * @param defaultTimeout Default timeout for requests. + * @param defaultTimeout Default timeout for requests. */ final class JettyRestClient( client: HttpClient, @@ -53,9 +55,9 @@ final class JettyRestClient( * The handler supports both regular responses and streaming responses, allowing for * incremental processing of large payloads through Observable streams. * - * @param baseUrl Base URL for the REST service + * @param baseUrl Base URL for the REST service * @param customMaxResponseLength Optional maximum response length override for non-streamed responses - * @param customTimeout Optional timeout override + * @param customTimeout Optional timeout override * @return A handler that can process REST requests with streaming capabilities */ def asHandleRequestWithStreaming( @@ -107,8 +109,7 @@ final class JettyRestClient( } bodyOpt.mapOr( { - // TODO streaming error handling client-side - callback(Failure(new Exception(s"Unsupported content type $contentTypeOpt"))) + callback(Failure(unsupportedContentTypeError(contentTypeOpt))) }, body => { this.collectToBuffer = false @@ -127,17 +128,19 @@ final class JettyRestClient( override def onContent(response: Response, chunk: Content.Chunk, demander: Runnable): Unit = if (collectToBuffer) super.onContent(response, chunk, demander) - else - if (chunk == Content.Chunk.EOF) { - rawContentSubject.onComplete() - } else { - val buf = chunk.getByteBuffer - val arr = new Array[Byte](buf.remaining) - buf.get(arr) - publishSubject.subscription // wait for subscription - .flatMapNow(_ => rawContentSubject.onNext(arr)) - .mapNow(_ => demander.run()) + else if (chunk == Content.Chunk.EOF) { + rawContentSubject.onComplete() + } else { + val buf = chunk.getByteBuffer + val arr = new Array[Byte](buf.remaining) + buf.get(arr) + publishSubject.subscription // wait for subscription + .flatMapNow(_ => rawContentSubject.onNext(arr)) + .mapNow { + case Ack.Continue => demander.run() + case Ack.Stop => () } + } override def onComplete(result: Result): Unit = if (result.isSucceeded) { @@ -169,9 +172,9 @@ final class JettyRestClient( * Creates a [[RawRest.HandleRequest]] which handles standard REST requests by buffering the entire response. * This does not support streaming responses. * - * @param baseUrl The base URL for the REST service. + * @param baseUrl The base URL for the REST service. * @param customMaxResponseLength Optional override for the maximum response length. - * @param customTimeout Optional override for the request timeout. + * @param customTimeout Optional override for the request timeout. * @return A `RawRest.HandleRequest` that buffers responses. */ def asHandleRequest( @@ -262,6 +265,10 @@ final class JettyRestClient( object JettyRestClient { final val DefaultMaxResponseLength = 2 * 1024 * 1024 final val DefaultTimeout = 10.seconds + final val Streaming: HttpErrorException = plain(400, "HTTP stream failure") + + def unsupportedContentTypeError(contentType: Opt[String]): HttpErrorException = + plain(400, s"Unsupported streaming Content-Type = ${contentType.getOrElse("null")}", new UnsupportedOperationException()) @explicitGenerics def apply[RestApi: RawRest.AsRealRpc : RestMetadata]( diff --git a/rest/src/main/scala/io/udash/rest/companions.scala b/rest/src/main/scala/io/udash/rest/companions.scala index bfea30d58..a23430d14 100644 --- a/rest/src/main/scala/io/udash/rest/companions.scala +++ b/rest/src/main/scala/io/udash/rest/companions.scala @@ -54,8 +54,18 @@ abstract class RestServerApiCompanion[Implicits, Real](protected val implicits: implicit final lazy val restMetadata: RestMetadata[Real] = inst(implicits, this).metadata implicit final lazy val restAsRaw: RawRest.AsRawRpc[Real] = inst(implicits, this).asRaw + + /** + * Maintained for backward compatibility with non-streaming clients. + * Converts the real API implementation into a request handler without streaming support. + */ final def asHandleRequest(real: Real): RawRest.HandleRequest = RawRest.asHandleRequest(real) + /** + * Converts the real API implementation into a request handler with streaming capabilities. + */ + final def asHandleRequestWithStreaming(real: Real): RawRest.HandleRequestWithStreaming = + RawRest.asHandleRequestWithStreaming(real) } /** @see [[io.udash.rest.RestApiCompanion RestApiCompanion]] */ @@ -68,6 +78,8 @@ abstract class RestServerOpenApiCompanion[Implicits, Real](protected val implici final def asHandleRequest(real: Real): RawRest.HandleRequest = RawRest.asHandleRequest(real) + final def asHandleRequestWithStreaming(real: Real): RawRest.HandleRequestWithStreaming = + RawRest.asHandleRequestWithStreaming(real) } /** @@ -103,6 +115,8 @@ abstract class RestOpenApiCompanion[Implicits, Real](protected val implicits: Im RawRest.fromHandleRequest(handleRequest) final def asHandleRequest(real: Real): RawRest.HandleRequest = RawRest.asHandleRequest(real) + final def asHandleRequestWithStreaming(real: Real): RawRest.HandleRequestWithStreaming = + RawRest.asHandleRequestWithStreaming(real) } trait PolyRestApiFullInstances[T[_[_]]] { @@ -136,6 +150,8 @@ abstract class RestServerApiImplCompanion[Implicits, Real](protected val implici final def asHandleRequest(real: Real): RawRest.HandleRequest = RawRest.asHandleRequest(real) + final def asHandleRequestWithStreaming(real: Real): RawRest.HandleRequestWithStreaming = + RawRest.asHandleRequestWithStreaming(real) } /** @@ -150,7 +166,6 @@ abstract class RestServerOpenApiImplCompanion[Implicits, Real](protected val imp final def asHandleRequest(real: Real): RawRest.HandleRequest = RawRest.asHandleRequest(real) - final def asHandleRequestWithStreaming(real: Real): RawRest.HandleRequestWithStreaming = RawRest.asHandleRequestWithStreaming(real) } diff --git a/rest/src/main/scala/io/udash/rest/implicits.scala b/rest/src/main/scala/io/udash/rest/implicits.scala index 997f1b0ab..6d8931b98 100644 --- a/rest/src/main/scala/io/udash/rest/implicits.scala +++ b/rest/src/main/scala/io/udash/rest/implicits.scala @@ -1,15 +1,15 @@ package io.udash package rest -import com.avsystem.commons._ +import com.avsystem.commons.* import com.avsystem.commons.meta.Fallback import com.avsystem.commons.misc.ImplicitNotFound import com.avsystem.commons.rpc.{AsRaw, AsRawReal, AsReal, InvalidRpcCall} import com.avsystem.commons.serialization.json.{JsonStringInput, JsonStringOutput} import com.avsystem.commons.serialization.{GenCodec, GenKeyCodec} import io.udash.rest.openapi.{OpenApiMetadata, RestSchema} +import io.udash.rest.raw.* import io.udash.rest.raw.RawRest.FromTask -import io.udash.rest.raw._ import monix.eval.Task import monix.execution.Scheduler diff --git a/rest/src/test/scala/io/udash/rest/CompilationErrorsTest.scala b/rest/src/test/scala/io/udash/rest/CompilationErrorsTest.scala index 96d2f1ccd..294a4536b 100644 --- a/rest/src/test/scala/io/udash/rest/CompilationErrorsTest.scala +++ b/rest/src/test/scala/io/udash/rest/CompilationErrorsTest.scala @@ -2,6 +2,8 @@ package io.udash package rest import io.udash.testing.CompilationErrorAssertions +import monix.eval.Task +import monix.reactive.Observable import scala.concurrent.Future import org.scalatest.funsuite.AnyFunSuite @@ -17,8 +19,6 @@ class CompilationErrorsTest extends AnyFunSuite with CompilationErrorAssertions def meth(par: Any): Future[Unit] } - // TODO streaming add streaming tests - test("missing serializer for parameter") { val error = norm(typeErrorFor("object Api extends DefaultRestApiCompanion[MissingSerializerForParam]")) assert(error == @@ -134,4 +134,52 @@ class CompilationErrorsTest extends AnyFunSuite with CompilationErrorAssertions | Cannot serialize Unit into StreamedRestResponse, because: | Cannot serialize Unit into io.udash.rest.raw.StreamedBody, appropriate AsRaw instance not found""".stripMargin) } -} + + + trait MissingObservableSerializerForResult { + @GET def streamMeth(): Observable[Any] + } + + test("missing serializer for Observable result element") { + val error = norm(typeErrorFor("object Api extends DefaultRestServerApiImplCompanion[MissingObservableSerializerForResult]")) + assert(error == + """cannot translate between trait MissingObservableSerializerForResult and trait RawRest: + |problem with method streamMeth: + | * it cannot be translated into an HTTP GET method: + | monix.reactive.Observable[Any] is not a valid result type because: + | Cannot serialize monix.reactive.Observable[Any] into RestResponse, because: + | Cannot serialize monix.reactive.Observable[Any] into HttpBody, because: + | Cannot serialize monix.reactive.Observable[Any] into JsonValue, because: + | No GenCodec found for monix.reactive.Observable[Any] + | * it cannot be translated into an HTTP GET stream method: + | monix.reactive.Observable[Any] is not a valid result type because: + | Cannot serialize monix.reactive.Observable[Any] into StreamedRestResponse, because: + | Cannot serialize Any into StreamedBody, because: + | Cannot serialize Any into JsonValue, because: + | No GenCodec found for Any""".stripMargin) + } + + trait MissingTaskObservableSerializerForResult { + @GET def taskStreamMeth(): Task[Observable[Any]] + } + + test("missing serializer for Task[Observable] result element") { + val error = norm(typeErrorFor("object Api extends DefaultRestApiCompanion[MissingTaskObservableSerializerForResult]")) + assert(error == + """cannot translate between trait MissingTaskObservableSerializerForResult and trait RawRest: + |problem with method taskStreamMeth: + | * it cannot be translated into an HTTP GET method: + | monix.eval.Task[monix.reactive.Observable[Any]] is not a valid result type because: + | Cannot serialize monix.reactive.Observable[Any] into RestResponse, because: + | Cannot serialize monix.reactive.Observable[Any] into HttpBody, because: + | Cannot serialize monix.reactive.Observable[Any] into JsonValue, because: + | No GenCodec found for monix.reactive.Observable[Any] + | * it cannot be translated into an HTTP GET stream method: + | monix.eval.Task[monix.reactive.Observable[Any]] is not a valid result type because: + | Cannot serialize monix.reactive.Observable[Any] into StreamedRestResponse, because: + | Cannot serialize Any into StreamedBody, because: + | Cannot serialize Any into JsonValue, because: + | No GenCodec found for Any""".stripMargin) + } + +} \ No newline at end of file diff --git a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala index d46b6c6ce..01dbe3842 100644 --- a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala @@ -2,10 +2,8 @@ package io.udash package rest import io.udash.rest.raw.HttpErrorException -import monix.eval.Task import monix.reactive.Observable -import java.util.concurrent.atomic.AtomicBoolean import scala.concurrent.duration.* trait StreamingRestTestApi { @@ -42,8 +40,7 @@ object StreamingRestTestApi extends DefaultRestApiCompanion[StreamingRestTestApi Observable.fromIterable(Range(0, 3)).map { i => if (i < 2) RestEntity(RestEntityId(i.toString), "first") else throw HttpErrorException.Streaming - } - + } override def delayedStream(size: Int, delayMillis: Long): Observable[Int] = { Observable.fromIterable(Range(0, size)) diff --git a/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala b/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala index e8c4367af..f01f409fa 100644 --- a/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala @@ -357,5 +357,15 @@ class ServerImplApiTest extends AnyFunSuite with ScalaFutures { expectedCode = 200 ) } + + test("streaming GET call to non-streaming endpoint") { + val params = RestParameters( + path = PlainValue.decodePath("/streamingNumbers"), + query = Mapping.create("count" -> PlainValue("5")) + ) + val request = RestRequest(HttpMethod.GET, params, HttpBody.Empty) + val response = RestResponse(200, IMapping.empty, HttpBody.json(JsonValue("[1,2,3,4,5]"))) + assertRawExchange(request, response) + } } From 06adaaeb1d3f028698a5172914e1295030a02e57 Mon Sep 17 00:00:00 2001 From: Krzysztof Maliszewski Date: Sat, 12 Apr 2025 11:28:28 +0200 Subject: [PATCH 076/162] Add documentation --- guide/guide/.js/src/main/assets/pages/rest.md | 209 ++++++++++++++++-- .../io/udash/rest/raw/RestResponse.scala | 2 +- 2 files changed, 191 insertions(+), 20 deletions(-) diff --git a/guide/guide/.js/src/main/assets/pages/rest.md b/guide/guide/.js/src/main/assets/pages/rest.md index 21fce10c1..6f2965ff4 100644 --- a/guide/guide/.js/src/main/assets/pages/rest.md +++ b/guide/guide/.js/src/main/assets/pages/rest.md @@ -703,7 +703,7 @@ derived from `GenCodec` instance. Ultimately, if you don't want to use `Future`s, you may replace it with some other asynchronous wrapper type, e.g. Monix Task or some IO monad. -See [supporting result containers other than `Future`](#supporting-result-containers-other-than-future). +See [supporting result containers other than `Future`](#supporting-result-containers-other-than-future), [streaming serialization workflow](#streaming-serialization-workflow). ### Customizing serialization @@ -1042,35 +1042,55 @@ When a client makes a request to a streaming endpoint: This approach allows processing of potentially unlimited amounts of data with minimal memory footprint on both the client and server. -### Streaming Types +#### Streaming response types -Udash REST supports two main streaming content types: +Streaming endpoints return data through a `StreamedRestResponse` instead of a regular `RestResponse`. This special +response type contains a `StreamedBody` which delivers content incrementally, rather than all at once. -1. **JSON Lists** - A stream of JSON objects sent as a JSON array `[{...}, {...}, ...]` -2. **Raw Binary** - A stream of binary data chunks, for files or other binary content +The framework supports two primary types of streaming responses: -### Implementation Details +1. **JSON Lists** - For streaming regular objects (any type `T` with a valid `RestSchema`), the content is delivered + as a stream of JSON values with `application/json` content type. Each element in the `Observable` is serialized + to JSON individually, allowing the client to process items as they arrive. -- Under the hood, streaming is implemented using Monix `Observable` -- Binary streams are transmitted as raw byte arrays -- JSON streams are automatically serialized/deserialized between JSON and your data types -- The server can control batch size to optimize network usage versus memory consumption +2. **Binary Streams** - For streaming binary data (`Observable[Array[Byte]]`), the content is delivered as a raw binary + stream with `application/octet-stream` content type. This is particularly useful for large file downloads or + real-time binary data processing. -### Handling Large Response Collections +#### Streaming serialization workflow -When dealing with large collections, streaming is preferable to loading everything in memory: +When a method returns an `Observable[T]`, the serialization flow is: -```scala -// Without streaming - entire list is loaded in memory -def getAllItems(): Future[List[Item]] +1. Each element of type `T` is serialized using the appropriate `AsRaw[JsonValue, T]` instance +2. These elements are delivered incrementally as part of a `StreamedBody.JsonList` +3. The client can process the items as they arrive, without waiting for the entire stream to complete + +For binary data (`Observable[Array[Byte]]`), each byte array chunk is directly sent through a `StreamedBody.RawBinary` +without additional transformation. + +#### Customizing streaming serialization -// With streaming - items are processed incrementally -def streamAllItems(): Observable[Item] +Just as with regular responses, you can customize how streaming responses are serialized. For instance, you might want +to provide a custom instance of `AsRaw[StreamedBody, Observable[T]]` for a specific type: + +```scala +// Custom serialization for streaming a specialized data type +implicit def customStreamingFormat[T: CustomFormat]: AsRaw[StreamedBody, Observable[T]] = + obs => StreamedBody.JsonList(obs.map(customToJsonValue)) ``` -The streaming version allows processing data incrementally, which is crucial for very large datasets that might exceed available memory. +#### Compatibility with non-streaming clients -## Error Handling with Streaming +For backward compatibility with clients that don't support streaming, the framework provides automatic conversion from +streaming responses to standard responses using `StreamedRestResponse.fallbackToRestResponse`. This materialization +process collects all elements from the stream and combines them into a single response: + +- For JSON streams, elements are collected into a JSON array +- For binary streams, byte arrays are concatenated + +However, this conversion loses the streaming benefits, so it's best used only when necessary. + +### Error Handling Streaming endpoints handle errors similarly to regular endpoints. When an error occurs during streaming: @@ -1093,6 +1113,76 @@ client.streamItems("") This allows graceful handling of errors that might occur during streaming operations. +### Advanced Streaming Patterns + +You can also use more advanced patterns for streaming responses: + +#### Task of Observable + +You can return a `Task` that resolves to an `Observable` when you need to perform some asynchronous work before starting the stream: + +```scala +import monix.eval.Task +import monix.reactive.Observable + +trait AdvancedStreamingApi { + /** Returns a Task that resolves to an Observable stream */ + def streamWithInitialProcessing(id: String): Task[Observable[DataPoint]] +} +object AdvancedStreamingApi extends DefaultRestApiCompanion[AdvancedStreamingApi] +``` + +Implementation example: + +```scala +class AdvancedStreamingApiImpl extends AdvancedStreamingApi { + def streamWithInitialProcessing(id: String): Task[Observable[DataPoint]] = + // First perform some async initialization work + Task.delay { + println(s"Starting stream for $id") + // Then return the actual stream + Observable.interval(1.second) + .map(i => DataPoint(id, i, System.currentTimeMillis())) + } +} +``` + +#### Custom Streaming Types + +You can create custom types with streaming capabilities by defining appropriate serialization in their companion objects: + +```scala +import monix.reactive.Observable + +// Custom wrapper around a stream of values +case class DataStream[T](source: Observable[T], metadata: Map[String, String]) + +object DataStream { + // Define how to serialize DataStream to StreamedBody + implicit def dataStreamAsRawReal[T](implicit jsonAsRaw: AsRaw[JsonValue, T]): AsRawReal[StreamedBody, DataStream[T]] = + AsRawReal.create( + // Serialization: DataStream -> StreamedBody + stream => StreamedBody.JsonList(stream.source.map(jsonAsRaw.asRaw)), + // Deserialization: StreamedBody -> DataStream + body => { + val elements = StreamedBody.castOrFail[StreamedBody.JsonList](body).elements + DataStream(elements.map(jsonAsReal.asReal), Map.empty) + } + ) +} + +trait CustomStreamingApi { + /** Returns a custom streaming type */ + def getDataStream(query: String): DataStream[SearchResult] + + /** Returns a Task that produces a custom streaming type */ + def prepareAndStreamData(id: String): Task[DataStream[DataPoint]] +} +object CustomStreamingApi extends DefaultRestApiCompanion[CustomStreamingApi] +``` + +This approach allows you to include additional metadata or context with your streams while maintaining the streaming behavior. + ## Generating OpenAPI 3.0 specifications [OpenAPI](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.2.md) is an open standard for describing @@ -1388,7 +1478,88 @@ Because multiple REST HTTP methods may have the same path, adjusters are collect are applied on the associated Path Item Object. When path item adjuster is applied on a [prefix method](#prefix-methods), it will apply to all Path Item Objects associated with result of this prefix method. +### OpenAPI for Streaming Endpoints + +When using Udash REST's streaming capabilities, OpenAPI specifications are automatically generated to reflect the streaming nature of the endpoints. This section describes how streaming responses are represented in OpenAPI documents. + +#### Streaming Response Schema + +Methods that return `Observable[T]` are automatically recognized as streaming endpoints. In the generated OpenAPI document, these endpoints are represented as arrays of the type `T`: + +```scala +trait StreamingApi { + // This will be documented as an array of Item in OpenAPI + def streamItems(filter: String): Observable[Item] +} +``` + +For a streaming endpoint returning `Observable[Item]`, the generated schema will represent this as an array of `Item` objects. The framework automatically wraps the element type in an array schema to indicate that multiple items may be delivered over time. + +#### Supported Streaming Formats + +The OpenAPI document correctly describes the available media types for streaming responses: + +1. **JSON Lists** - When streaming regular objects, they're represented as a JSON array in the schema. + This is reflected in the OpenAPI document with content type `application/json`. + +2. **Binary Streams** - When streaming `Array[Byte]` (binary data), the content type in the OpenAPI document + will be `application/octet-stream`. + +Example schema representation for a streaming endpoint: + +```json +{ + "paths": { + "/streamItems": { + "get": { + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Item" + } + } + } + } + } + } + } + } + } +} +``` + +#### Customizing OpenAPI for Streaming Endpoints + +You can use the same annotation-based customization mechanisms discussed earlier to modify the OpenAPI documentation for streaming endpoints: + +```scala +trait StreamingApi { + @description("Streams items matching the filter criteria") + @adjustOperation(op => op.copy( + responses = op.responses.copy( + byStatusCode = op.responses.byStatusCode + ( + 200 -> RefOr(Response( + description = "A stream of matching items that may be processed incrementally", + content = op.responses.byStatusCode(200).value.content + )) + ) + ) + )) + def streamItems(filter: String): Observable[Item] + + @description("Streams binary file data") + def downloadFile(id: String): Observable[Array[Byte]] +} +``` + ### Limitations - Current representation of OpenAPI document does not support [specification extensions](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.2.md#specificationExtensions). + +- While the OpenAPI specification doesn't have native support for true streaming semantics, Udash REST represents streaming endpoints as array responses. This is the most accurate representation within the constraints of the OpenAPI specification format. Consumers of your API should understand that these array responses may be delivered incrementally rather than all at once, especially for potentially large datasets. diff --git a/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala b/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala index 578c277bd..cd6f6c46e 100644 --- a/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala +++ b/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala @@ -19,7 +19,7 @@ sealed trait AbstractRestResponse { final def isSuccess: Boolean = code >= 200 && code < 300 } -/** Standard REST response containing a status code, headers, and a body. The body is loaded fully in memory as an HttpBody. */ +/** Standard REST response containing a status code, headers, and a body. The body is loaded fully in memory as an [[HttpBody]]. */ final case class RestResponse( code: Int, headers: IMapping[PlainValue], From 666371f96ad8705e39b33e1096d1f3f8f6fe5cea Mon Sep 17 00:00:00 2001 From: Krzysztof Maliszewski Date: Mon, 14 Apr 2025 09:27:09 +0200 Subject: [PATCH 077/162] Add unit tests for advanced streaming types and non-streaming implementation --- guide/guide/.js/src/main/assets/pages/rest.md | 3 -- .../scala/io/udash/rest/RestApiTest.scala | 38 ++++++++++++++++++- .../io/udash/rest/StreamingRestTestApi.scala | 34 ++++++++++++++++- 3 files changed, 70 insertions(+), 5 deletions(-) diff --git a/guide/guide/.js/src/main/assets/pages/rest.md b/guide/guide/.js/src/main/assets/pages/rest.md index 6f2965ff4..99b106831 100644 --- a/guide/guide/.js/src/main/assets/pages/rest.md +++ b/guide/guide/.js/src/main/assets/pages/rest.md @@ -1172,9 +1172,6 @@ object DataStream { } trait CustomStreamingApi { - /** Returns a custom streaming type */ - def getDataStream(query: String): DataStream[SearchResult] - /** Returns a Task that produces a custom streaming type */ def prepareAndStreamData(id: String): Task[DataStream[DataPoint]] } diff --git a/rest/src/test/scala/io/udash/rest/RestApiTest.scala b/rest/src/test/scala/io/udash/rest/RestApiTest.scala index 5e631c429..c37b9a745 100644 --- a/rest/src/test/scala/io/udash/rest/RestApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/RestApiTest.scala @@ -4,7 +4,7 @@ package rest import cats.implicits.catsSyntaxTuple2Semigroupal import com.avsystem.commons.* import com.avsystem.commons.misc.ScalaDurationExtensions.durationIntOps -import io.udash.rest.raw.{HttpErrorException, RawRest} +import io.udash.rest.raw.RawRest import io.udash.testing.AsyncUdashSharedTest import monix.eval.Task import monix.execution.Scheduler @@ -194,4 +194,40 @@ trait StreamingRestApiTestScenarios extends RestApiTest { } } } + + "streaming with non-streaming client" in { + val standardProxy = RawRest.fromHandleRequest[StreamingRestTestApi](clientHandle) + standardProxy.simpleStream(3).toListL.materialize.runToFuture.map { + case Failure(exception: UnsupportedOperationException) => + assert(exception.getMessage == "Streaming unsupported by the client") + case Failure(otherException) => + fail(s"Expected UnsupportedOperationException but got ${otherException.getClass.getName}: ${otherException.getMessage}") + case Success(_) => + fail("Expected UnsupportedOperationException but operation succeeded") + } + } + + "task of observable stream" in { + val testTask = for { + proxyResults <- streamingProxy.delayedStreamTask(3, 50).flatMap(_.toListL) + implResults <- streamingImpl.delayedStreamTask(3, 50).flatMap(_.toListL) + } yield { + assert(proxyResults.map(mkDeep) == implResults.map(mkDeep)) + } + testTask.runToFuture + } + + "custom stream task" in { + val testTask = for { + proxyResults <- streamingProxy.customStreamTask(3) + implResults <- streamingImpl.customStreamTask(3) + proxyObs <- proxyResults.source.toListL + implObs <- implResults.source.toListL + } yield { + assert(proxyResults.metadata == implResults.metadata) + assert(proxyObs == implObs) + } + testTask.runToFuture + } + } diff --git a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala index 01dbe3842..8f8c3dc8d 100644 --- a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala @@ -1,11 +1,29 @@ package io.udash package rest -import io.udash.rest.raw.HttpErrorException +import com.avsystem.commons.rpc.AsRawReal +import com.avsystem.commons.serialization.json.JsonStringOutput +import io.udash.rest.openapi.RestSchema +import io.udash.rest.raw.{HttpErrorException, JsonValue, StreamedBody} +import monix.eval.Task import monix.reactive.Observable import scala.concurrent.duration.* +case class DataStream(source: Observable[Int], metadata: Map[String, String]) + +object DataStream { + implicit def schema: RestSchema[DataStream] = ??? + implicit def dataStreamAsRawReal: AsRawReal[StreamedBody, DataStream] = + AsRawReal.create( + stream => StreamedBody.JsonList(stream.source.map(i => JsonValue(JsonStringOutput.write(i)))), + { + case StreamedBody.JsonList(e, c) => DataStream(e.map(_.value.toInt), Map.empty) + case _ => ??? + } + ) +} + trait StreamingRestTestApi { @GET def simpleStream(size: Int): Observable[Int] @@ -16,6 +34,10 @@ trait StreamingRestTestApi { @POST def errorStream(@Query immediate: Boolean): Observable[RestEntity] @GET def delayedStream(@Query size: Int, @Query delayMillis: Long): Observable[Int] + + @GET def delayedStreamTask(@Query size: Int, @Query delayMillis: Long): Task[Observable[Int]] + @GET def customStreamTask(@Query size: Int): Task[DataStream] + } object StreamingRestTestApi extends DefaultRestApiCompanion[StreamingRestTestApi] { @@ -47,5 +69,15 @@ object StreamingRestTestApi extends DefaultRestApiCompanion[StreamingRestTestApi .zip(Observable.intervalAtFixedRate(delayMillis.millis, delayMillis.millis)) .map(_._1) } + + override def delayedStreamTask(size: Int, delayMillis: Long): Task[Observable[Int]] = + Task.delay(delayedStream(size, delayMillis)) + + override def customStreamTask(size: Int): Task[DataStream] = Task { + DataStream( + Observable.fromIterable(Range(0, size)), + Map.empty + ) + } } } \ No newline at end of file From 40b30325ba9abd387f285ededde72cf8a0f032b2 Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Tue, 15 Apr 2025 15:27:42 +0200 Subject: [PATCH 078/162] Fix implicit messages --- .../io/udash/rest/raw/RestResponse.scala | 30 ++++++++++++++----- .../io/udash/rest/raw/StreamedBody.scala | 4 +-- .../io/udash/rest/CompilationErrorsTest.scala | 13 +++----- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala b/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala index cd6f6c46e..0dc21f4c9 100644 --- a/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala +++ b/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala @@ -55,12 +55,14 @@ object RestResponse extends RestResponseLowPrio { } implicit def taskLikeFromResponseTask[F[_], T]( - implicit fromTask: FromTask[F], fromResponse: AsReal[RestResponse, T] + implicit fromTask: FromTask[F], + fromResponse: AsReal[RestResponse, T], ): AsReal[Task[RestResponse], Try[F[T]]] = rawTask => Success(fromTask.fromTask(rawTask.map(fromResponse.asReal))) implicit def taskLikeToResponseTask[F[_], T]( - implicit taskLike: TaskLike[F], asResponse: AsRaw[RestResponse, T] + implicit taskLike: TaskLike[F], + asResponse: AsRaw[RestResponse, T], ): AsRaw[Task[RestResponse], Try[F[T]]] = _.fold(Task.raiseError, ft => Task.from(ft).map(asResponse.asRaw)).recoverHttpError @@ -70,13 +72,13 @@ object RestResponse extends RestResponseLowPrio { @implicitNotFound("${F}[${T}] is not a valid result type because:\n#{forResponseType}") implicit def effAsyncAsRealNotFound[F[_], T](implicit fromAsync: TaskLike[F], - forResponseType: ImplicitNotFound[AsReal[RestResponse, T]] + forResponseType: ImplicitNotFound[AsReal[RestResponse, T]], ): ImplicitNotFound[AsReal[Task[RestResponse], Try[F[T]]]] = ImplicitNotFound() @implicitNotFound("${F}[${T}] is not a valid result type because:\n#{forResponseType}") implicit def effAsyncAsRawNotFound[F[_], T](implicit toAsync: TaskLike[F], - forResponseType: ImplicitNotFound[AsRaw[RestResponse, T]] + forResponseType: ImplicitNotFound[AsRaw[RestResponse, T]], ): ImplicitNotFound[AsRaw[Task[RestResponse], Try[F[T]]]] = ImplicitNotFound() // following two implicits provide nice error messages when result type of HTTP method is totally wrong @@ -185,12 +187,14 @@ object StreamedRestResponse extends StreamedRestResponseLowPrio { } implicit def taskLikeFromResponseTask[F[_], T]( - implicit fromTask: FromTask[F], fromResponse: AsReal[StreamedRestResponse, T] + implicit fromTask: FromTask[F], + fromResponse: AsReal[StreamedRestResponse, T], ): AsReal[Task[StreamedRestResponse], Try[F[T]]] = rawTask => Success(fromTask.fromTask(rawTask.map(fromResponse.asReal))) implicit def taskLikeToResponseTask[F[_], T]( - implicit taskLike: TaskLike[F], asResponse: AsRaw[StreamedRestResponse, T] + implicit taskLike: TaskLike[F], + asResponse: AsRaw[StreamedRestResponse, T], ): AsRaw[Task[StreamedRestResponse], Try[F[T]]] = _.fold(Task.raiseError, ft => Task.from(ft).map(asResponse.asRaw)).recoverHttpError @@ -204,8 +208,8 @@ object StreamedRestResponse extends StreamedRestResponseLowPrio { ): AsRaw[Task[StreamedRestResponse], Try[Observable[T]]] = _.fold(Task.raiseError, ft => Task.eval(ft).map(asResponse.asRaw)).recoverHttpError - // following two implicits provide nice error messages when serialization is lacking for HTTP method result - // while the async wrapper is fine (e.g. Future) + // following implicits provide nice error messages when serialization is lacking for HTTP method result + // while the async wrapper is fine @implicitNotFound("${F}[${T}] is not a valid result type because:\n#{forResponseType}") implicit def effAsyncAsRealNotFound[F[_], T](implicit @@ -219,6 +223,16 @@ object StreamedRestResponse extends StreamedRestResponseLowPrio { forResponseType: ImplicitNotFound[AsRaw[StreamedRestResponse, T]] ): ImplicitNotFound[AsRaw[Task[StreamedRestResponse], Try[F[T]]]] = ImplicitNotFound() + @implicitNotFound("Observable[${T}] is not a valid result type because:\n#{forResponseType}") + implicit def observableAsRealNotFound[T](implicit + forResponseType: ImplicitNotFound[AsReal[StreamedBody, Observable[T]]] + ): ImplicitNotFound[AsReal[Task[StreamedRestResponse], Try[Observable[T]]]] = ImplicitNotFound() + + @implicitNotFound("Observable[${T}] is not a valid result type because:\n#{forResponseType}") + implicit def observableAsRawNotFound[T](implicit + forResponseType: ImplicitNotFound[AsRaw[StreamedBody, Observable[T]]] + ): ImplicitNotFound[AsRaw[Task[StreamedRestResponse], Try[Observable[T]]]] = ImplicitNotFound() + // following two implicits provide nice error messages when result type of HTTP method is totally wrong @implicitNotFound("#{forResponseType}") diff --git a/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala b/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala index 6a6d5120a..8957c68a0 100644 --- a/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala +++ b/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala @@ -98,12 +98,12 @@ trait StreamedBodyLowPrio { this: StreamedBody.type => implicit def bodyJsonListAsReal[T](implicit jsonAsReal: AsReal[JsonValue, T]): AsReal[StreamedBody, Observable[T]] = v => StreamedBody.castOrFail[StreamedBody.JsonList](v).elements.map(jsonAsReal.asReal) - @implicitNotFound("Cannot deserialize ${T} from StreamedBody, because:\n#{forJson}") + @implicitNotFound("Cannot deserialize Observable[${T}] from StreamedBody, because:\n#{forJson}") implicit def asRealNotFound[T]( implicit forJson: ImplicitNotFound[AsReal[JsonValue, T]] ): ImplicitNotFound[AsReal[StreamedBody, Observable[T]]] = ImplicitNotFound() - @implicitNotFound("Cannot serialize ${T} into StreamedBody, because:\n#{forJson}") + @implicitNotFound("Cannot serialize Observable[${T}] into StreamedBody, because:\n#{forJson}") implicit def asRawNotFound[T]( implicit forJson: ImplicitNotFound[AsRaw[JsonValue, T]] ): ImplicitNotFound[AsRaw[StreamedBody, Observable[T]]] = ImplicitNotFound() diff --git a/rest/src/test/scala/io/udash/rest/CompilationErrorsTest.scala b/rest/src/test/scala/io/udash/rest/CompilationErrorsTest.scala index 294a4536b..724de50ea 100644 --- a/rest/src/test/scala/io/udash/rest/CompilationErrorsTest.scala +++ b/rest/src/test/scala/io/udash/rest/CompilationErrorsTest.scala @@ -146,15 +146,10 @@ class CompilationErrorsTest extends AnyFunSuite with CompilationErrorAssertions """cannot translate between trait MissingObservableSerializerForResult and trait RawRest: |problem with method streamMeth: | * it cannot be translated into an HTTP GET method: - | monix.reactive.Observable[Any] is not a valid result type because: - | Cannot serialize monix.reactive.Observable[Any] into RestResponse, because: - | Cannot serialize monix.reactive.Observable[Any] into HttpBody, because: - | Cannot serialize monix.reactive.Observable[Any] into JsonValue, because: - | No GenCodec found for monix.reactive.Observable[Any] + | monix.reactive.Observable[Any] is not a valid result type of HTTP REST method - it must be a Future | * it cannot be translated into an HTTP GET stream method: - | monix.reactive.Observable[Any] is not a valid result type because: - | Cannot serialize monix.reactive.Observable[Any] into StreamedRestResponse, because: - | Cannot serialize Any into StreamedBody, because: + | Observable[Any] is not a valid result type because: + | Cannot serialize Observable[Any] into StreamedBody, because: | Cannot serialize Any into JsonValue, because: | No GenCodec found for Any""".stripMargin) } @@ -177,7 +172,7 @@ class CompilationErrorsTest extends AnyFunSuite with CompilationErrorAssertions | * it cannot be translated into an HTTP GET stream method: | monix.eval.Task[monix.reactive.Observable[Any]] is not a valid result type because: | Cannot serialize monix.reactive.Observable[Any] into StreamedRestResponse, because: - | Cannot serialize Any into StreamedBody, because: + | Cannot serialize Observable[Any] into StreamedBody, because: | Cannot serialize Any into JsonValue, because: | No GenCodec found for Any""".stripMargin) } From 1943dd660e325feafdab755e4337306e50829249 Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Tue, 15 Apr 2025 15:49:16 +0200 Subject: [PATCH 079/162] Fix OpenAPI schema test --- .../test/resources/StreamingRestTestApi.json | 79 +++++++++++++++++++ .../io/udash/rest/StreamingRestTestApi.scala | 26 +++--- 2 files changed, 92 insertions(+), 13 deletions(-) diff --git a/rest/.jvm/src/test/resources/StreamingRestTestApi.json b/rest/.jvm/src/test/resources/StreamingRestTestApi.json index 8ae1e2f59..292c90d44 100644 --- a/rest/.jvm/src/test/resources/StreamingRestTestApi.json +++ b/rest/.jvm/src/test/resources/StreamingRestTestApi.json @@ -24,6 +24,35 @@ } } }, + "/customStreamTask": { + "get": { + "operationId": "customStreamTask", + "parameters": [ + { + "name": "size", + "in": "query", + "required": true, + "explode": false, + "schema": { + "type": "integer", + "format": "int32" + } + } + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DataStream" + } + } + } + } + } + } + }, "/delayedStream": { "get": { "operationId": "delayedStream", @@ -67,6 +96,49 @@ } } }, + "/delayedStreamTask": { + "get": { + "operationId": "delayedStreamTask", + "parameters": [ + { + "name": "size", + "in": "query", + "required": true, + "explode": false, + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "delayMillis", + "in": "query", + "required": true, + "explode": false, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + } + } + } + } + } + } + }, "/errorStream": { "post": { "operationId": "errorStream", @@ -159,6 +231,13 @@ ], "components": { "schemas": { + "DataStream": { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + }, "RestEntity": { "type": "object", "description": "REST entity", diff --git a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala index 8f8c3dc8d..efb7633e2 100644 --- a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala @@ -1,8 +1,7 @@ package io.udash package rest -import com.avsystem.commons.rpc.AsRawReal -import com.avsystem.commons.serialization.json.JsonStringOutput +import com.avsystem.commons.rpc.{AsRaw, AsRawReal, AsReal} import io.udash.rest.openapi.RestSchema import io.udash.rest.raw.{HttpErrorException, JsonValue, StreamedBody} import monix.eval.Task @@ -10,17 +9,19 @@ import monix.reactive.Observable import scala.concurrent.duration.* -case class DataStream(source: Observable[Int], metadata: Map[String, String]) +final case class DataStream(source: Observable[Int], metadata: Map[String, String]) + +object DataStream extends GenCodecRestImplicits { + implicit def schema: RestSchema[DataStream] = + RestSchema.create(res => RestSchema.seqSchema[Seq, Int].createSchema(res), "DataStream") -object DataStream { - implicit def schema: RestSchema[DataStream] = ??? implicit def dataStreamAsRawReal: AsRawReal[StreamedBody, DataStream] = AsRawReal.create( - stream => StreamedBody.JsonList(stream.source.map(i => JsonValue(JsonStringOutput.write(i)))), - { - case StreamedBody.JsonList(e, c) => DataStream(e.map(_.value.toInt), Map.empty) - case _ => ??? - } + stream => StreamedBody.JsonList(stream.source.map(AsRaw[JsonValue, Int].asRaw)), + rawBody => { + val list = StreamedBody.castOrFail[StreamedBody.JsonList](rawBody) + DataStream(list.elements.map(AsReal[JsonValue, Int].asReal), Map.empty) + }, ) } @@ -36,8 +37,8 @@ trait StreamingRestTestApi { @GET def delayedStream(@Query size: Int, @Query delayMillis: Long): Observable[Int] @GET def delayedStreamTask(@Query size: Int, @Query delayMillis: Long): Task[Observable[Int]] - @GET def customStreamTask(@Query size: Int): Task[DataStream] + @GET def customStreamTask(@Query size: Int): Task[DataStream] } object StreamingRestTestApi extends DefaultRestApiCompanion[StreamingRestTestApi] { @@ -64,11 +65,10 @@ object StreamingRestTestApi extends DefaultRestApiCompanion[StreamingRestTestApi else throw HttpErrorException.Streaming } - override def delayedStream(size: Int, delayMillis: Long): Observable[Int] = { + override def delayedStream(size: Int, delayMillis: Long): Observable[Int] = Observable.fromIterable(Range(0, size)) .zip(Observable.intervalAtFixedRate(delayMillis.millis, delayMillis.millis)) .map(_._1) - } override def delayedStreamTask(size: Int, delayMillis: Long): Task[Observable[Int]] = Task.delay(delayedStream(size, delayMillis)) From 0a695099b7cc37cbd36cd7c9246b8e3c56bc0fa1 Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Tue, 15 Apr 2025 16:33:26 +0200 Subject: [PATCH 080/162] Adjust streaming tests --- .../test/resources/StreamingRestTestApi.json | 70 +++++++++++++------ .../rest/openapi/OpenApiGenerationTest.scala | 1 + .../io/udash/rest/jetty/JettyRestClient.scala | 16 +++-- .../scala/io/udash/rest/raw/RestRequest.scala | 1 - .../scala/io/udash/rest/RestApiTest.scala | 67 +++++++++++------- .../io/udash/rest/StreamingRestTestApi.scala | 47 ++++++++++--- .../io/udash/rest/raw/ServerImplApiTest.scala | 1 - 7 files changed, 136 insertions(+), 67 deletions(-) diff --git a/rest/.jvm/src/test/resources/StreamingRestTestApi.json b/rest/.jvm/src/test/resources/StreamingRestTestApi.json index 292c90d44..704d0ca88 100644 --- a/rest/.jvm/src/test/resources/StreamingRestTestApi.json +++ b/rest/.jvm/src/test/resources/StreamingRestTestApi.json @@ -24,9 +24,9 @@ } } }, - "/customStreamTask": { + "/customStream": { "get": { - "operationId": "customStreamTask", + "operationId": "customStream", "parameters": [ { "name": "size", @@ -45,7 +45,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/DataStream" + "$ref": "#/components/schemas/CustomStream" } } } @@ -53,9 +53,9 @@ } } }, - "/delayedStream": { + "/customStreamTask": { "get": { - "operationId": "delayedStream", + "operationId": "customStreamTask", "parameters": [ { "name": "size", @@ -66,16 +66,6 @@ "type": "integer", "format": "int32" } - }, - { - "name": "delayMillis", - "in": "query", - "required": true, - "explode": false, - "schema": { - "type": "integer", - "format": "int64" - } } ], "responses": { @@ -84,11 +74,7 @@ "content": { "application/json": { "schema": { - "type": "array", - "items": { - "type": "integer", - "format": "int32" - } + "$ref": "#/components/schemas/DataStream" } } } @@ -96,9 +82,9 @@ } } }, - "/delayedStreamTask": { + "/delayedStream": { "get": { - "operationId": "delayedStreamTask", + "operationId": "delayedStream", "parameters": [ { "name": "size", @@ -222,6 +208,39 @@ } } } + }, + "/streamTask": { + "get": { + "operationId": "streamTask", + "parameters": [ + { + "name": "size", + "in": "query", + "required": true, + "explode": false, + "schema": { + "type": "integer", + "format": "int32" + } + } + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + } + } + } + } + } + } } }, "servers": [ @@ -231,6 +250,13 @@ ], "components": { "schemas": { + "CustomStream": { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + }, "DataStream": { "type": "array", "items": { diff --git a/rest/.jvm/src/test/scala/io/udash/rest/openapi/OpenApiGenerationTest.scala b/rest/.jvm/src/test/scala/io/udash/rest/openapi/OpenApiGenerationTest.scala index fc6949021..ab4c1c507 100644 --- a/rest/.jvm/src/test/scala/io/udash/rest/openapi/OpenApiGenerationTest.scala +++ b/rest/.jvm/src/test/scala/io/udash/rest/openapi/OpenApiGenerationTest.scala @@ -8,6 +8,7 @@ import scala.io.Source import org.scalatest.funsuite.AnyFunSuite class OpenApiGenerationTest extends AnyFunSuite { + test("openapi for RestTestApi") { val openapi = RestTestApi.openapiMetadata.openapi( Info("Test API", "0.1", description = "Some test REST API"), diff --git a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala index 4ea5b2b65..85357b0ef 100644 --- a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala +++ b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala @@ -4,9 +4,7 @@ package rest.jetty import com.avsystem.commons.* import com.avsystem.commons.annotation.explicitGenerics import com.avsystem.commons.serialization.json.{JsonReader, JsonStringInput} -import io.udash.rest.jetty.JettyRestClient.unsupportedContentTypeError import io.udash.rest.raw.* -import io.udash.rest.raw.HttpErrorException.plain import io.udash.rest.util.Utils import io.udash.utils.URLEncoder import monix.eval.Task @@ -103,13 +101,13 @@ final class JettyRestClient( Observable .fromIterator(Task.eval(input.readList().iterator(_.asInstanceOf[JsonStringInput].readRawJson()))) .map(JsonValue(_)) - }.onErrorFallbackTo(Observable.raiseError(HttpErrorException.Streaming)), + }.onErrorFallbackTo(Observable.raiseError(JettyRestClient.Streaming)), charset = charset, ) } bodyOpt.mapOr( { - callback(Failure(unsupportedContentTypeError(contentTypeOpt))) + callback(Failure(JettyRestClient.unsupportedContentTypeError(contentTypeOpt))) }, body => { this.collectToBuffer = false @@ -265,10 +263,14 @@ final class JettyRestClient( object JettyRestClient { final val DefaultMaxResponseLength = 2 * 1024 * 1024 final val DefaultTimeout = 10.seconds - final val Streaming: HttpErrorException = plain(400, "HTTP stream failure") + final val Streaming = HttpErrorException.plain(400, "HTTP stream failure") - def unsupportedContentTypeError(contentType: Opt[String]): HttpErrorException = - plain(400, s"Unsupported streaming Content-Type = ${contentType.getOrElse("null")}", new UnsupportedOperationException()) + private def unsupportedContentTypeError(contentType: Opt[String]): HttpErrorException = + HttpErrorException.plain( + code = 400, + message = s"Unsupported streaming Content-Type${contentType.mapOr("", c => s" = $c")}", + cause = new UnsupportedOperationException, + ) @explicitGenerics def apply[RestApi: RawRest.AsRealRpc : RestMetadata]( diff --git a/rest/src/main/scala/io/udash/rest/raw/RestRequest.scala b/rest/src/main/scala/io/udash/rest/raw/RestRequest.scala index 860e6edb2..eafaaf09e 100644 --- a/rest/src/main/scala/io/udash/rest/raw/RestRequest.scala +++ b/rest/src/main/scala/io/udash/rest/raw/RestRequest.scala @@ -56,7 +56,6 @@ case class HttpErrorException(code: Int, payload: HttpBody = HttpBody.Empty, cau object HttpErrorException { def plain(code: Int, message: String, cause: Throwable = null): HttpErrorException = HttpErrorException(code, HttpBody.plain(message), cause) - val Streaming: HttpErrorException = plain(400, "HTTP stream failure") } final case class RestRequest(method: HttpMethod, parameters: RestParameters, body: HttpBody) { diff --git a/rest/src/test/scala/io/udash/rest/RestApiTest.scala b/rest/src/test/scala/io/udash/rest/RestApiTest.scala index c37b9a745..f9ea5fc13 100644 --- a/rest/src/test/scala/io/udash/rest/RestApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/RestApiTest.scala @@ -165,6 +165,42 @@ trait StreamingRestApiTestScenarios extends RestApiTest { testStream(_.binaryStream()) } + "task of observable stream" in { + val testTask = for { + proxyResults <- streamingProxy.streamTask(size = 3).flatMap(_.toListL) + implResults <- streamingImpl.streamTask(size = 3).flatMap(_.toListL) + } yield { + assert(proxyResults.map(mkDeep) == implResults.map(mkDeep)) + } + testTask.runToFuture + } + + "custom stream task" in { + val testTask = for { + proxyResults <- streamingProxy.customStreamTask(3) + implResults <- streamingImpl.customStreamTask(3) + proxyObs <- proxyResults.source.toListL + implObs <- implResults.source.toListL + } yield { + assert(proxyResults.metadata == implResults.metadata) + assert(proxyObs == implObs) + } + testTask.runToFuture + } + + "custom stream" in { + val testTask = for { + proxyResults <- streamingProxy.customStream(3) + implResults <- streamingImpl.customStream(3) + proxyObs <- proxyResults.source.toListL + implObs <- implResults.source.toListL + } yield { + assert(proxyResults.code == implResults.code) + assert(proxyObs == implObs) + } + testTask.runToFuture + } + "immediate stream error" in { testStream(_.errorStream(immediate = true)) } @@ -184,15 +220,16 @@ trait StreamingRestApiTestScenarios extends RestApiTest { val timeoutTask = streamTask.timeout(500.millis).materialize - timeoutTask.runToFuture.map { result => + timeoutTask.map { result => assert(result.isFailure, "Stream should have failed due to timeout") result match { case Failure(ex) => assert(ex.isInstanceOf[TimeoutException], s"Expected TimeoutException, but got $ex") succeed - case Success(_) => fail("Stream succeeded unexpectedly despite timeout") + case Success(_) => + fail("Stream succeeded unexpectedly despite timeout") } - } + }.runToFuture } "streaming with non-streaming client" in { @@ -206,28 +243,4 @@ trait StreamingRestApiTestScenarios extends RestApiTest { fail("Expected UnsupportedOperationException but operation succeeded") } } - - "task of observable stream" in { - val testTask = for { - proxyResults <- streamingProxy.delayedStreamTask(3, 50).flatMap(_.toListL) - implResults <- streamingImpl.delayedStreamTask(3, 50).flatMap(_.toListL) - } yield { - assert(proxyResults.map(mkDeep) == implResults.map(mkDeep)) - } - testTask.runToFuture - } - - "custom stream task" in { - val testTask = for { - proxyResults <- streamingProxy.customStreamTask(3) - implResults <- streamingImpl.customStreamTask(3) - proxyObs <- proxyResults.source.toListL - implObs <- implResults.source.toListL - } yield { - assert(proxyResults.metadata == implResults.metadata) - assert(proxyObs == implObs) - } - testTask.runToFuture - } - } diff --git a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala index efb7633e2..0db03abbf 100644 --- a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala @@ -3,7 +3,7 @@ package rest import com.avsystem.commons.rpc.{AsRaw, AsRawReal, AsReal} import io.udash.rest.openapi.RestSchema -import io.udash.rest.raw.{HttpErrorException, JsonValue, StreamedBody} +import io.udash.rest.raw.* import monix.eval.Task import monix.reactive.Observable @@ -15,7 +15,7 @@ object DataStream extends GenCodecRestImplicits { implicit def schema: RestSchema[DataStream] = RestSchema.create(res => RestSchema.seqSchema[Seq, Int].createSchema(res), "DataStream") - implicit def dataStreamAsRawReal: AsRawReal[StreamedBody, DataStream] = + implicit val dataStreamAsRawReal: AsRawReal[StreamedBody, DataStream] = AsRawReal.create( stream => StreamedBody.JsonList(stream.source.map(AsRaw[JsonValue, Int].asRaw)), rawBody => { @@ -25,6 +25,25 @@ object DataStream extends GenCodecRestImplicits { ) } +final case class CustomStream(source: Observable[Int], code: Int) +object CustomStream extends GenCodecRestImplicits { + implicit def schema: RestSchema[CustomStream] = + RestSchema.create(res => RestSchema.seqSchema[Seq, Int].createSchema(res), "CustomStream") + + implicit val customStreamAsRawReal: AsRawReal[StreamedRestResponse, CustomStream] = + AsRawReal.create( + stream => StreamedRestResponse( + code = stream.code, + headers = IMapping.empty, + body = StreamedBody.JsonList(stream.source.map(AsRaw[JsonValue, Int].asRaw)), + ), + rawResponse => { + val list = StreamedBody.castOrFail[StreamedBody.JsonList](rawResponse.body) + CustomStream(list.elements.map(AsReal[JsonValue, Int].asReal), rawResponse.code) + }, + ) +} + trait StreamingRestTestApi { @GET def simpleStream(size: Int): Observable[Int] @@ -32,13 +51,16 @@ trait StreamingRestTestApi { @POST def binaryStream(): Observable[Array[Byte]] + @streamingResponseBatchSize(3) @POST def errorStream(@Query immediate: Boolean): Observable[RestEntity] @GET def delayedStream(@Query size: Int, @Query delayMillis: Long): Observable[Int] - @GET def delayedStreamTask(@Query size: Int, @Query delayMillis: Long): Task[Observable[Int]] + @GET def streamTask(@Query size: Int): Task[Observable[Int]] @GET def customStreamTask(@Query size: Int): Task[DataStream] + + @GET def customStream(@Query size: Int): Task[CustomStream] } object StreamingRestTestApi extends DefaultRestApiCompanion[StreamingRestTestApi] { @@ -62,7 +84,7 @@ object StreamingRestTestApi extends DefaultRestApiCompanion[StreamingRestTestApi else Observable.fromIterable(Range(0, 3)).map { i => if (i < 2) RestEntity(RestEntityId(i.toString), "first") - else throw HttpErrorException.Streaming + else throw HttpErrorException.plain(400, "bad stream") } override def delayedStream(size: Int, delayMillis: Long): Observable[Int] = @@ -70,14 +92,21 @@ object StreamingRestTestApi extends DefaultRestApiCompanion[StreamingRestTestApi .zip(Observable.intervalAtFixedRate(delayMillis.millis, delayMillis.millis)) .map(_._1) - override def delayedStreamTask(size: Int, delayMillis: Long): Task[Observable[Int]] = - Task.delay(delayedStream(size, delayMillis)) + override def streamTask(size: Int): Task[Observable[Int]] = + Task.eval(Observable.fromIterable(Range(0, size))) override def customStreamTask(size: Int): Task[DataStream] = Task { DataStream( - Observable.fromIterable(Range(0, size)), - Map.empty + source = Observable.fromIterable(Range(0, size)), + metadata = Map.empty + ) + } + + override def customStream(size: Int): Task[CustomStream] = Task { + CustomStream( + source = Observable.fromIterable(Range(0, size)), + code = 200, ) } } -} \ No newline at end of file +} diff --git a/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala b/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala index f01f409fa..0c606f2c7 100644 --- a/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala @@ -368,4 +368,3 @@ class ServerImplApiTest extends AnyFunSuite with ScalaFutures { assertRawExchange(request, response) } } - From 7debb1bb606670ac4e1e406eef8da70227b17c76 Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Thu, 17 Apr 2025 12:43:31 +0200 Subject: [PATCH 081/162] Add streaming chapter to guide --- guide/guide/.js/src/main/scala/io/udash/web/guide/init.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/guide/guide/.js/src/main/scala/io/udash/web/guide/init.scala b/guide/guide/.js/src/main/scala/io/udash/web/guide/init.scala index e08d23b7a..34cea3787 100644 --- a/guide/guide/.js/src/main/scala/io/udash/web/guide/init.scala +++ b/guide/guide/.js/src/main/scala/io/udash/web/guide/init.scala @@ -61,6 +61,7 @@ object Context { markdownLink(RestState, "Serialization"), markdownLink(RestState, "API evolution"), markdownLink(RestState, "Implementing backends"), + markdownLink(RestState, "Streaming Support"), markdownLink(RestState, "Generating OpenAPI 3.0 specifications"), )), MenuContainer("Extensions", Seq( From 604708c08502dc8cda568b38256408958f800b45 Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Thu, 17 Apr 2025 12:50:43 +0200 Subject: [PATCH 082/162] Batching only for Json list, more tests --- .../src/main/scala/io/udash/rest/RestServlet.scala | 11 +++++------ .../scala/io/udash/rest/ServletBasedRestApiTest.scala | 2 +- .../scala/io/udash/rest/jetty/JettyRestClient.scala | 2 -- rest/src/main/scala/io/udash/rest/annotations.scala | 10 +++++++++- .../main/scala/io/udash/rest/raw/RestResponse.scala | 1 - .../main/scala/io/udash/rest/raw/StreamedBody.scala | 3 ++- rest/src/test/scala/io/udash/rest/RestApiTest.scala | 8 ++++++-- .../scala/io/udash/rest/StreamingRestTestApi.scala | 1 + .../test/scala/io/udash/rest/raw/RawRestTest.scala | 4 ++-- 9 files changed, 26 insertions(+), 16 deletions(-) diff --git a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala index e08c821b8..f9a9db039 100644 --- a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala +++ b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala @@ -22,7 +22,7 @@ object RestServlet { final val DefaultHandleTimeout = 30.seconds final val DefaultMaxPayloadSize = 16 * 1024 * 1024L // 16MB final val CookieHeader = "Cookie" - final val DefaultStreamingBatchSize = 1 + final val DefaultStreamingBatchSize = 100 private final val BufferSize = 8192 /** @@ -32,7 +32,7 @@ object RestServlet { * @param handleTimeout maximum time the servlet will wait for results returned by REST API implementation * @param maxPayloadSize maximum acceptable incoming payload size, in bytes; * if exceeded, `413 Payload Too Large` response will be sent back - * @param defaultStreamingBatchSize default batch size for [[StreamedRestResponse]] + * @param defaultStreamingBatchSize default batch when streaming [[StreamedBody.JsonList]] */ @explicitGenerics def apply[RestApi: RawRest.AsRawRpc : RestMetadata]( apiImpl: RestApi, @@ -130,15 +130,14 @@ class RestServlet( case binary: StreamedBody.RawBinary => response.setContentType(binary.contentType) binary.content - .bufferTumbling(stream.customBatchSize.getOrElse(defaultStreamingBatchSize)) - .consumeWith(Consumer.foreach { batch => - batch.foreach(e => response.getOutputStream.write(e)) + .consumeWith(Consumer.foreach { chunk => + response.getOutputStream.write(chunk) response.getOutputStream.flush() }) case jsonList: StreamedBody.JsonList => response.setContentType(jsonList.contentType) jsonList.elements - .bufferTumbling(stream.customBatchSize.getOrElse(defaultStreamingBatchSize)) + .bufferTumbling(jsonList.customBatchSize.getOrElse(defaultStreamingBatchSize)) .switchIfEmpty(Observable(Seq.empty)) .zipWithIndex .consumeWith(Consumer.foreach { case (batch, idx) => diff --git a/rest/.jvm/src/test/scala/io/udash/rest/ServletBasedRestApiTest.scala b/rest/.jvm/src/test/scala/io/udash/rest/ServletBasedRestApiTest.scala index 8ced322fb..fe4191852 100644 --- a/rest/.jvm/src/test/scala/io/udash/rest/ServletBasedRestApiTest.scala +++ b/rest/.jvm/src/test/scala/io/udash/rest/ServletBasedRestApiTest.scala @@ -14,7 +14,7 @@ abstract class ServletBasedRestApiTest extends RestApiTest with UsesHttpServer { protected def setupServer(server: Server): Unit = { val servlet = new RestServlet(serverHandle, serverTimeout, maxPayloadSize) - val streamingServlet = new RestServlet(streamingServerHandle, serverTimeout, maxPayloadSize) + val streamingServlet = new RestServlet(streamingServerHandle, serverTimeout, maxPayloadSize, defaultStreamingBatchSize = 1) val handler = new ServletContextHandler() handler.addServlet(new ServletHolder(servlet), "/api/*") handler.addServlet(new ServletHolder(streamingServlet), "/stream-api/*") diff --git a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala index 85357b0ef..7b13cd0ce 100644 --- a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala +++ b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala @@ -115,7 +115,6 @@ final class JettyRestClient( code = response.getStatus, headers = parseHeaders(response), body = body, - customBatchSize = Opt.Empty, ) callback(Success(restResponse)) } @@ -151,7 +150,6 @@ final class JettyRestClient( code = httpResp.getStatus, headers = parseHeaders(httpResp), body = StreamedBody.fromHttpBody(parseHttpBody(httpResp, this)), - customBatchSize = Opt.Empty, ) callback(Success(restResponse)) } else { diff --git a/rest/src/main/scala/io/udash/rest/annotations.scala b/rest/src/main/scala/io/udash/rest/annotations.scala index a19ce1a05..ccfbb9682 100644 --- a/rest/src/main/scala/io/udash/rest/annotations.scala +++ b/rest/src/main/scala/io/udash/rest/annotations.scala @@ -367,7 +367,15 @@ class addResponseHeader(name: String, value: String) extends ResponseAdjuster wi override def adjustResponse(response: StreamedRestResponse): StreamedRestResponse = response.header(name, value) } +/** + * Annotation which may be applied on REST API methods to change streaming batch size when [[StreamedBody.JsonList]] + * bodies are used. It has no effect when applied to other body types or non-streaming methods. + */ class streamingResponseBatchSize(size: Int) extends StreamedResponseAdjuster { override def adjustResponse(response: StreamedRestResponse): StreamedRestResponse = - response.copy(customBatchSize = Opt.some(size)) + response.body match { + case jsonList: StreamedBody.JsonList => + response.copy(body = jsonList.copy(customBatchSize = Opt.some(size))) + case _ => response + } } diff --git a/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala b/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala index 0dc21f4c9..836d0dbbb 100644 --- a/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala +++ b/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala @@ -121,7 +121,6 @@ final case class StreamedRestResponse( code: Int, headers: IMapping[PlainValue], body: StreamedBody, - customBatchSize: Opt[Int] = Opt.Empty, ) extends AbstractRestResponse { def header(name: String, value: String): StreamedRestResponse = diff --git a/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala b/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala index 8957c68a0..d8891dbc9 100644 --- a/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala +++ b/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala @@ -2,7 +2,7 @@ package io.udash package rest.raw import com.avsystem.commons.annotation.explicitGenerics -import com.avsystem.commons.misc.ImplicitNotFound +import com.avsystem.commons.misc.{ImplicitNotFound, Opt} import com.avsystem.commons.rpc.{AsRaw, AsRawReal, AsReal} import com.avsystem.commons.serialization.GenCodec.ReadFailure import monix.reactive.Observable @@ -49,6 +49,7 @@ object StreamedBody extends StreamedBodyLowPrio { final case class JsonList( elements: Observable[JsonValue], charset: String = HttpBody.Utf8Charset, + customBatchSize: Opt[Int] = Opt.Empty, ) extends NonEmpty { val contentType: String = s"${HttpBody.JsonType};charset=$charset" diff --git a/rest/src/test/scala/io/udash/rest/RestApiTest.scala b/rest/src/test/scala/io/udash/rest/RestApiTest.scala index f9ea5fc13..2458a2f32 100644 --- a/rest/src/test/scala/io/udash/rest/RestApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/RestApiTest.scala @@ -146,14 +146,18 @@ trait RestApiTestScenarios extends RestApiTest { } } -// TODO streaming MORE tests: cancellation, timeouts, errors, errors after sending a few elements, custom format, slow source observable +// TODO streaming MORE tests: cancellation trait StreamingRestApiTestScenarios extends RestApiTest { "empty GET stream" in { testStream(_.simpleStream(0)) } - "trivial GET stream" in { + "trivial GET stream - single batch" in { + testStream(_.simpleStream(1)) + } + + "trivial GET stream - multi batch" in { testStream(_.simpleStream(5)) } diff --git a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala index 0db03abbf..9e8b8ee83 100644 --- a/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/StreamingRestTestApi.scala @@ -45,6 +45,7 @@ object CustomStream extends GenCodecRestImplicits { } trait StreamingRestTestApi { + @streamingResponseBatchSize(3) @GET def simpleStream(size: Int): Observable[Int] @GET def jsonStream: Observable[RestEntity] diff --git a/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala b/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala index 97047e624..5e57d8866 100644 --- a/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala +++ b/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala @@ -122,7 +122,7 @@ class RawRestTest extends AnyFunSuite with ScalaFutures with Matchers { val bodyReprTask: Task[String] = resp.body match { case StreamedBody.Empty => Task.now("") - case StreamedBody.JsonList(elements, charset) => + case StreamedBody.JsonList(elements, charset, _) => elements.toListL.map { list => s" application/json;charset=$charset\n[${list.map(_.value).mkString(",")}]" } @@ -506,7 +506,7 @@ class RawRestTest extends AnyFunSuite with ScalaFutures with Matchers { HttpBody.Empty ) whenReady(serverHandleWithStreaming(request).runToFuture) { - case StreamedRestResponse(code, headers, body, batchSize) => + case StreamedRestResponse(code, headers, body) => code shouldBe 200 val elements = castOrFail[StreamedBody.JsonList](body).elements whenReady(elements.toListL.runToFuture) { e => From 32495d027ee235e3e27297f0a9e6fe2b54b2111a Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Thu, 17 Apr 2025 13:08:12 +0200 Subject: [PATCH 083/162] More streaming mentions in REST docs --- guide/guide/.js/src/main/assets/pages/rest.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/guide/guide/.js/src/main/assets/pages/rest.md b/guide/guide/.js/src/main/assets/pages/rest.md index 99b106831..55db3cde1 100644 --- a/guide/guide/.js/src/main/assets/pages/rest.md +++ b/guide/guide/.js/src/main/assets/pages/rest.md @@ -680,7 +680,7 @@ serializable as `HttpBody`. ### Result serialization -Result type of every REST API method is wrapped into `Try` (in case the method throws an exception) +Result type of every non-streaming REST API method is wrapped into `Try` (in case the method throws an exception) and translated into `Task[RestResponse]`. This means that the macro engine looks for an implicit instance of `AsRaw[Task[RestResponse], Try[R]]` and `AsReal[Task[RestResponse], Try[R]]` for every HTTP method with result type `R`. @@ -703,7 +703,10 @@ derived from `GenCodec` instance. Ultimately, if you don't want to use `Future`s, you may replace it with some other asynchronous wrapper type, e.g. Monix Task or some IO monad. -See [supporting result containers other than `Future`](#supporting-result-containers-other-than-future), [streaming serialization workflow](#streaming-serialization-workflow). +See [supporting result containers other than `Future`](#supporting-result-containers-other-than-future). + +See [streaming serialization workflow](#streaming-serialization-workflow) for details on `monix.reactive.Observable` +support in streaming REST API methods. ### Customizing serialization @@ -946,8 +949,8 @@ computation which yields a `RestResponse` when run. ### Implementing a server -An existing implementation of REST API trait can be easily turned into a `HandleRequest` -function using `RawRest.asHandleRequest`. +An existing implementation of REST API trait can be easily turned into a function using +`RawRest.asHandleRequest` or `RawRest.asHandleRequestWithStreaming` (for server-side streaming support). Therefore, the only thing you need to do to expose your REST API trait as an actual web service it to turn `HandleRequest` function into a server. This is usually just a matter of translating native HTTP request into @@ -961,7 +964,8 @@ for an example implementation. If you already have a `HandleRequest` function, you can easily turn it into an implementation of desired REST API trait using `RawRest.fromHandleRequest`. This implementation is a macro-generated proxy which translates actual -method calls into invocations of provided `HandleRequest` function. +method calls into invocations of provided `HandleRequest` function. Use `RawRest.fromHandleRequestWithStreaming` for +client-side streaming support. Therefore, the only thing you need to to in order to wrap a native HTTP client into a REST API trait instance is to turn this native HTTP client into a `HandleRequest` function. @@ -1195,7 +1199,7 @@ materialized by a macro. You can then use it to generate OpenAPI specification d import io.udash.rest._ trait MyRestApi { - ... + // ... } object MyRestApi extends DefaultRestApiCompanion[MyRestApi] @@ -1496,7 +1500,7 @@ For a streaming endpoint returning `Observable[Item]`, the generated schema will The OpenAPI document correctly describes the available media types for streaming responses: -1. **JSON Lists** - When streaming regular objects, they're represented as a JSON array in the schema. +1. **JSON Lists** - When streaming regular JSON objects, they're represented as a JSON array in the schema. This is reflected in the OpenAPI document with content type `application/json`. 2. **Binary Streams** - When streaming `Array[Byte]` (binary data), the content type in the OpenAPI document From fcb0c9387f8e0610a5dded34a93d16e329459b5b Mon Sep 17 00:00:00 2001 From: Krzysztof Maliszewski Date: Tue, 22 Apr 2025 10:43:03 +0200 Subject: [PATCH 084/162] Add mid stream error handling and RestServletTest --- project/Dependencies.scala | 2 + .../scala/io/udash/rest/RestServlet.scala | 15 +- .../scala/io/udash/rest/RestServletTest.scala | 352 ++++++++++++++++++ .../scala/io/udash/rest/RestApiTest.scala | 4 +- 4 files changed, 365 insertions(+), 8 deletions(-) create mode 100644 rest/.jvm/src/test/scala/io/udash/rest/RestServletTest.scala diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 7550d1f26..cf242e0f7 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -37,6 +37,7 @@ object Dependencies { val svg4everybodyVersion = "2.1.9" val scalatestVersion = "3.2.19" + val mockitoVersion = "1.17.37" val scalaJsSecureRandomVersion = "1.0.0" // Tests only val bootstrap4Version = "4.1.3" val bootstrap4DatepickerVersion = "5.39.0" @@ -115,6 +116,7 @@ object Dependencies { "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion, "org.eclipse.jetty" % "jetty-server" % jettyVersion % Test, "org.eclipse.jetty.ee8" % "jetty-ee8-servlet" % jettyVersion % Test, + "org.mockito" %% "mockito-scala-scalatest" % mockitoVersion % Test, )) val restSjsDeps = restCrossDeps diff --git a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala index f9a9db039..b0701e05c 100644 --- a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala +++ b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala @@ -28,10 +28,10 @@ object RestServlet { /** * Wraps an implementation of some REST API trait into a Java Servlet. * - * @param apiImpl implementation of some REST API trait - * @param handleTimeout maximum time the servlet will wait for results returned by REST API implementation - * @param maxPayloadSize maximum acceptable incoming payload size, in bytes; - * if exceeded, `413 Payload Too Large` response will be sent back + * @param apiImpl implementation of some REST API trait + * @param handleTimeout maximum time the servlet will wait for results returned by REST API implementation + * @param maxPayloadSize maximum acceptable incoming payload size, in bytes; + * if exceeded, `413 Payload Too Large` response will be sent back * @param defaultStreamingBatchSize default batch when streaming [[StreamedBody.JsonList]] */ @explicitGenerics def apply[RestApi: RawRest.AsRawRpc : RestMetadata]( @@ -59,7 +59,7 @@ class RestServlet( scheduler: Scheduler ) extends HttpServlet with LazyLogging { - import RestServlet._ + import RestServlet.* override def service(request: HttpServletRequest, response: HttpServletResponse): Unit = { val asyncContext = request.startAsync() @@ -159,6 +159,9 @@ class RestServlet( }) .map(_ => response.getOutputStream.write("]".getBytes(jsonList.charset))) } + }.onErrorHandle { e => + logger.error(e.getMessage) + response.getOutputStream.close() } private def writeResponseBody(response: HttpServletResponse, rr: AbstractRestResponse): Task[Unit] = @@ -178,7 +181,7 @@ class RestServlet( private def writeResponse(response: HttpServletResponse, restResponse: RestResponse): Unit = { setResponseHeaders(response, restResponse.code, restResponse.headers) restResponse.body match { - case HttpBody.Empty => + case HttpBody.Empty => case neBody: HttpBody.NonEmpty => writeNonEmptyBody(response, neBody) } } diff --git a/rest/.jvm/src/test/scala/io/udash/rest/RestServletTest.scala b/rest/.jvm/src/test/scala/io/udash/rest/RestServletTest.scala new file mode 100644 index 000000000..7eb6ed6e2 --- /dev/null +++ b/rest/.jvm/src/test/scala/io/udash/rest/RestServletTest.scala @@ -0,0 +1,352 @@ +package io.udash +package rest + +import com.avsystem.commons.* +import monix.execution.Scheduler +import monix.reactive.Observable +import org.mockito.ArgumentMatchers.argThat +import org.mockito.Mockito.* +import org.scalatest.BeforeAndAfterEach +import org.scalatest.concurrent.{Eventually, ScalaFutures} +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Millis, Seconds, Span} + +import java.io.* +import javax.servlet.AsyncContext +import javax.servlet.http.{HttpServletRequest, HttpServletResponse} +import scala.concurrent.Future + +class RestServletTest extends AnyFunSuite with ScalaFutures with Matchers with BeforeAndAfterEach with Eventually { + implicit def scheduler: Scheduler = Scheduler.global + + implicit override val patienceConfig: PatienceConfig = PatienceConfig( + timeout = Span(5, Seconds), + interval = Span(50, Millis) + ) + + var request: HttpServletRequest = _ + var response: HttpServletResponse = _ + var asyncContext: AsyncContext = _ + var outputStream: ByteArrayOutputStream = _ + var writer: StringWriter = _ + var printWriter: PrintWriter = _ + var servlet: RestServlet = _ + + override def beforeEach(): Unit = { + super.beforeEach() + + request = mock(classOf[HttpServletRequest]) + response = mock(classOf[HttpServletResponse]) + asyncContext = mock(classOf[AsyncContext]) + outputStream = new ByteArrayOutputStream() + writer = new StringWriter() + printWriter = new PrintWriter(writer) + + when(request.startAsync()).thenReturn(asyncContext) + when(response.getOutputStream).thenReturn(spy(new ServletOutputStreamMock(outputStream))) + when(response.getWriter).thenReturn(printWriter) + + servlet = RestServlet[TestApi](new TestApiImpl(), + maxPayloadSize = 1024 * 1024 + ) + } + + def setupRequest( + method: String = "GET", + path: String, + contentType: String = null, + body: String = "", + queryString: String = null, + headers: Map[String, String] = Map.empty, + cookies: Array[javax.servlet.http.Cookie] = null + ): Unit = { + when(request.getMethod).thenReturn(method) + when(request.getRequestURI).thenReturn(path) + when(request.getQueryString).thenReturn(queryString) + when(request.getContextPath).thenReturn("") + when(request.getServletPath).thenReturn("") + when(request.getContentType).thenReturn(contentType) + + if (contentType != null && body.nonEmpty) { + when(request.getContentLengthLong).thenReturn(body.getBytes.length.toLong) + when(request.getReader).thenReturn(new BufferedReader(new StringReader(body))) + } else { + when(request.getContentLengthLong).thenReturn(-1L) + } + + if (headers.isEmpty) { + when(request.getHeaderNames).thenReturn(java.util.Collections.emptyEnumeration()) + } else { + val headerNames = java.util.Collections.enumeration(headers.keys.toList.asJava) + when(request.getHeaderNames).thenReturn(headerNames) + headers.foreach { case (name, value) => + when(request.getHeader(name)).thenReturn(value) + } + } + + when(request.getCookies).thenReturn(cookies) + } + + def setupGetRequest(path: String, queryString: String = null): Unit = { + setupRequest(path = path, queryString = queryString) + } + + def setupPostRequest(path: String, contentType: String, body: String): Unit = { + setupRequest(method = "POST", path = path, contentType = contentType, body = body) + } + + def verifyResponse(expectedStatus: Int, expectedContentType: Opt[String] = Opt.empty): Unit = { + verify(response).setStatus(expectedStatus) + expectedContentType.foreach { contentType => + verify(response).setContentType(argThat((argument: String) => argument.startsWith(contentType))) + } + verify(asyncContext).complete() + } + + test("GET request should return simple response") { + setupGetRequest("/hello", "name=TestUser") + + servlet.service(request, response) + eventually { + verifyResponse(200, Opt("application/json")) + outputStream.toString should include("Hello, TestUser") + } + } + + test("POST request should process JSON body") { + setupPostRequest( + "/echo", + "application/json;charset=utf-8", + """{"message":"Hello World"}""" + ) + + servlet.service(request, response) + + eventually { + verifyResponse(200, Opt("application/json")) + outputStream.toString should include("""Hello World""") + } + } + + test("Binary streaming should work correctly") { + setupGetRequest("/binary", "size=10") + + servlet.service(request, response) + + eventually { + verifyResponse(200, Opt("application/octet-stream")) + val bytes = outputStream.toByteArray + bytes.length should be(10) + outputStream.toString shouldEqual "A".repeat(10) + } + } + + test("Error response should be handled properly") { + setupGetRequest("/error") + + servlet.service(request, response) + + eventually { + verifyResponse(500, Opt("text/plain")) + writer.toString should include("Test error") + } + } + + test("PUT request should update a resource") { + setupRequest( + method = "PUT", + path = "/update/123", + contentType = "application/json;charset=utf-8", + body = """{"data":"new content"}""" + ) + + servlet.service(request, response) + + eventually { + verifyResponse(200, Opt("application/json")) + outputStream.toString should include("Updated 123") + } + } + + test("DELETE request should remove a resource") { + setupRequest(method = "DELETE", path = "/remove/123") + + servlet.service(request, response) + + eventually { + verifyResponse(204) + } + } + + test("Form body should be processed correctly") { + setupRequest( + method = "POST", + path = "/form", + contentType = "application/x-www-form-urlencoded;charset=utf-8", + body = "name=John%20Doe&age=30", + queryString = "id=user123" + ) + + servlet.service(request, response) + + eventually { + verifyResponse(200, Opt("application/json")) + outputStream.toString should include("Form: user123, John Doe, 30") + } + } + + test("Empty streaming response should be handled correctly") { + setupGetRequest("/emptyStream") + + servlet.service(request, response) + + eventually { + verifyResponse(200, Opt("application/json")) + outputStream.toString shouldEqual "[]" + } + } + + test("Cookie values should be processed correctly") { + val cookies = Array(new javax.servlet.http.Cookie("sessionId", "abc123")) + setupRequest(path = "/withCookie", cookies = cookies) + + servlet.service(request, response) + + eventually { + verifyResponse(200, Opt("application/json")) + outputStream.toString should include("Session: abc123") + } + } + + test("Request headers should be processed correctly") { + setupRequest( + path = "/withHeader", + headers = Map("X-Custom" -> "test-value") + ) + + servlet.service(request, response) + + eventually { + verifyResponse(200, Opt("application/json")) + outputStream.toString should include("Header: test-value") + } + } + + test("JSON streaming response should be delivered in chunks") { + setupGetRequest("/jsonBatched", "count=5") + + servlet.service(request, response) + + eventually { + verifyResponse(200, Opt("application/json")) + val output = outputStream.toString("UTF-8") + output shouldEqual "[{\"value\":1},{\"value\":2},{\"value\":3},{\"value\":4},{\"value\":5}]" + } + } + + test("Streaming error should be handled properly") { + setupGetRequest("/streamError", "failAt=3") + + servlet.service(request, response) + + eventually { + verifyResponse(200, Opt("application/json")) + outputStream.toString shouldEqual "[\"item-1\",\"item-2\"" + // closes HTTP connection on failure + verify(response.getOutputStream).close() + } + } + + test("Request exceeding payload limit should return 413") { + val smallServlet = RestServlet[TestApi](new TestApiImpl(), maxPayloadSize = 10) + + setupPostRequest("/echo", "application/json", "{\"message\":\"This payload is too long and should fail\"}") + + smallServlet.service(request, response) + + eventually { + verify(response).setStatus(413) + verify(asyncContext).complete() + } + } +} + +class ServletOutputStreamMock(baos: ByteArrayOutputStream) extends javax.servlet.ServletOutputStream { + def write(b: Int): Unit = baos.write(b) + def isReady: Boolean = true + def setWriteListener(writeListener: javax.servlet.WriteListener): Unit = {} +} + +trait TestApi { + @GET def hello(@Query name: String): Future[String] + @POST def echo(message: String): Future[String] + @PUT def update(@Path id: String, data: String): Future[String] + @DELETE def remove(@Path id: String): Future[Unit] + @GET def binary(@Query size: Int): Observable[Array[Byte]] + @GET def error: Future[String] + @streamingResponseBatchSize(1) + @GET def streamError(@Query failAt: Int): Observable[String] + @POST @FormBody def form(@Query id: String, name: String, age: Int): Future[String] + @GET def emptyStream: Observable[String] + @GET def longRunning(@Query seconds: Int): Future[String] + @GET def withCookie(@Cookie sessionId: String): Future[String] + @GET def withHeader(@Header("X-Custom") custom: String): Future[String] + @GET def jsonBatched(@Query count: Int): Observable[Map[String, Int]] +} +object TestApi extends DefaultRestApiCompanion[TestApi] + +class TestApiImpl extends TestApi { + implicit val ec: ExecutionContext = scala.concurrent.ExecutionContext.global + + def hello(name: String): Future[String] = + Future.successful(s"Hello, $name") + + def echo(message: String): Future[String] = + Future.successful(message) + + def update(id: String, data: String): Future[String] = + Future.successful(s"Updated $id with $data") + + def remove(id: String): Future[Unit] = + Future.successful(()) + + @streamingResponseBatchSize(1) + def numbers(count: Int): Observable[Int] = + Observable.range(1, count + 1).map(_.toInt) + + def binary(size: Int): Observable[Array[Byte]] = { + val chunk = "A".repeat(size).getBytes + Observable.pure(chunk) + } + + def error: Future[String] = + Future.failed(new RuntimeException("Test error")) + + def streamError(failAt: Int): Observable[String] = + Observable.range(1, 10).map(i => + if (i == failAt) throw new RuntimeException(s"Error at item $i") + else s"item-$i" + ) + + def form(id: String, name: String, age: Int): Future[String] = + Future.successful(s"Form: $id, $name, $age") + + def emptyStream: Observable[String] = + Observable.empty + + def longRunning(seconds: Int): Future[String] = + Future { + Thread.sleep(seconds * 1000) + s"Completed after $seconds seconds" + } + + def withCookie(sessionId: String): Future[String] = + Future.successful(s"Session: $sessionId") + + def withHeader(custom: String): Future[String] = + Future.successful(s"Header: $custom") + + def jsonBatched(count: Int): Observable[Map[String, Int]] = + Observable.range(1, count + 1).map(i => Map("value" -> i.toInt)) +} \ No newline at end of file diff --git a/rest/src/test/scala/io/udash/rest/RestApiTest.scala b/rest/src/test/scala/io/udash/rest/RestApiTest.scala index 2458a2f32..145ecca75 100644 --- a/rest/src/test/scala/io/udash/rest/RestApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/RestApiTest.scala @@ -205,11 +205,11 @@ trait StreamingRestApiTestScenarios extends RestApiTest { testTask.runToFuture } - "immediate stream error" in { + "immediate stream error" ignore { testStream(_.errorStream(immediate = true)) } - "mid-stream error" in { + "mid-stream error" ignore { testStream(_.errorStream(immediate = false)) } From 0bcc120ec003624014f003f7baa8e73d6d641802 Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Tue, 22 Apr 2025 17:49:50 +0200 Subject: [PATCH 085/162] Client-side cancellation --- .../io/udash/rest/jetty/JettyRestClient.scala | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala index 7b13cd0ce..44be2ebce 100644 --- a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala +++ b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala @@ -71,7 +71,10 @@ final class JettyRestClient( override def handleRequestStream(request: RestRequest): Task[StreamedRestResponse] = prepareRequest(baseUrl, timeout, request).flatMap { httpReq => - Task.async0 { (scheduler: Scheduler, callback: Callback[Throwable, StreamedRestResponse]) => + def cancelRequest: Task[Unit] = + Task(httpReq.abort(new CancellationException("Request cancelled")).discard) + + Task.cancelable0 { (scheduler: Scheduler, callback: Callback[Throwable, StreamedRestResponse]) => val listener = new BufferingResponseListener(maxResponseLength) { private var collectToBuffer: Boolean = true private lazy val publishSubject = PublishToOneSubject[Array[Byte]]() @@ -88,7 +91,7 @@ final class JettyRestClient( val mediaTypeOpt = contentTypeOpt.map(MimeTypes.getContentTypeWithoutCharset) val bodyOpt = mediaTypeOpt matchOpt { case Opt(HttpBody.OctetStreamType) => - StreamedBody.RawBinary(content = rawContentSubject) + StreamedBody.RawBinary(content = rawContentSubject.doOnSubscriptionCancel(cancelRequest)) case Opt(HttpBody.JsonType) => val charset = contentTypeOpt.map(MimeTypes.getCharsetFromContentType).getOrElse(HttpBody.Utf8Charset) // suboptimal - maybe "online" parsing is possible using Jackson / other lib without waiting for full content ? @@ -101,7 +104,9 @@ final class JettyRestClient( Observable .fromIterator(Task.eval(input.readList().iterator(_.asInstanceOf[JsonStringInput].readRawJson()))) .map(JsonValue(_)) - }.onErrorFallbackTo(Observable.raiseError(JettyRestClient.Streaming)), + } + .doOnSubscriptionCancel(cancelRequest) + .onErrorFallbackTo(Observable.raiseError(JettyRestClient.Streaming)), charset = charset, ) } @@ -134,9 +139,9 @@ final class JettyRestClient( publishSubject.subscription // wait for subscription .flatMapNow(_ => rawContentSubject.onNext(arr)) .mapNow { - case Ack.Continue => demander.run() - case Ack.Stop => () - } + case Ack.Continue => demander.run() + case Ack.Stop => () + } } override def onComplete(result: Result): Unit = @@ -160,7 +165,9 @@ final class JettyRestClient( } } httpReq.send(listener) - }.doOnCancel(Task(httpReq.abort(new CancellationException("Request cancelled")))) + + cancelRequest // see cats.effect#CancelToken + } } } From 2a711cbe1c05dd7fa33d8cac18770df74209e740 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 22 Apr 2025 19:12:04 +0000 Subject: [PATCH 086/162] Update circe-core, circe-parser to 0.14.13 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index dfcefb172..7c4caf501 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -20,7 +20,7 @@ object Dependencies { val atmosphereVersion = "2.7.15" val upickleVersion = "4.1.0" // Tests only - val circeVersion = "0.14.12" // Tests only + val circeVersion = "0.14.13" // Tests only val circeDerivationVersion = "0.13.0-M5" // Tests only val monixVersion = "3.4.1" // udash-rest only From b768a4d688f12db41151f62845ad73c6fe9807cc Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 22 Apr 2025 19:12:07 +0000 Subject: [PATCH 087/162] Update webdrivermanager to 6.1.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index dfcefb172..c34e5d93f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -43,7 +43,7 @@ object Dependencies { val momentJsVersion = "2.30.1" val seleniumVersion = "4.31.0" - val webDriverManagerVersion = "6.0.1" + val webDriverManagerVersion = "6.1.0" val scalaJsBenchmarkVersion = "0.10.0" val compilerPlugins = Def.setting(Seq( From ae478f4aaa39ced0be9f648167c1415649f5bbf1 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 22 Apr 2025 19:12:20 +0000 Subject: [PATCH 088/162] Update sbt-scalajs, scalajs-compiler, ... to 1.19.0 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 07fe5c046..22e3a457b 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -3,7 +3,7 @@ logLevel := Level.Warn libraryDependencies += "org.scala-js" %% "scalajs-env-selenium" % "1.1.1" libraryDependencies += "org.scala-js" %% "scalajs-env-jsdom-nodejs" % "1.1.0" -addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.18.2") +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.19.0") addSbtPlugin("org.scala-js" % "sbt-jsdependencies" % "1.0.2") addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.4") addSbtPlugin("org.jetbrains.scala" % "sbt-ide-settings" % "1.1.2") From c20e1bc8c14f0305c0d2a701b28b89bef07d6169 Mon Sep 17 00:00:00 2001 From: Krzysztof Maliszewski Date: Tue, 22 Apr 2025 21:23:56 +0200 Subject: [PATCH 089/162] Remove obsolete tests --- rest/src/test/scala/io/udash/rest/RestApiTest.scala | 9 --------- 1 file changed, 9 deletions(-) diff --git a/rest/src/test/scala/io/udash/rest/RestApiTest.scala b/rest/src/test/scala/io/udash/rest/RestApiTest.scala index 145ecca75..8a7868d88 100644 --- a/rest/src/test/scala/io/udash/rest/RestApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/RestApiTest.scala @@ -146,7 +146,6 @@ trait RestApiTestScenarios extends RestApiTest { } } -// TODO streaming MORE tests: cancellation trait StreamingRestApiTestScenarios extends RestApiTest { "empty GET stream" in { @@ -205,14 +204,6 @@ trait StreamingRestApiTestScenarios extends RestApiTest { testTask.runToFuture } - "immediate stream error" ignore { - testStream(_.errorStream(immediate = true)) - } - - "mid-stream error" ignore { - testStream(_.errorStream(immediate = false)) - } - "slow source stream" in { testStream(_.delayedStream(size = 3, delayMillis = 100)) } From 96d91439e5468a0961951c177c752b65ccc8cfbd Mon Sep 17 00:00:00 2001 From: Krzysztof Maliszewski Date: Wed, 23 Apr 2025 14:44:24 +0200 Subject: [PATCH 090/162] Enhance streaming support in REST API: add error handling and update documentation --- guide/guide/.js/src/main/assets/pages/rest.md | 7 ++++--- .../main/scala/io/udash/rest/RestServlet.scala | 9 ++++++--- .../scala/io/udash/rest/RestDataCompanion.scala | 16 ++++++++++++++-- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/guide/guide/.js/src/main/assets/pages/rest.md b/guide/guide/.js/src/main/assets/pages/rest.md index 55db3cde1..fcc44de3e 100644 --- a/guide/guide/.js/src/main/assets/pages/rest.md +++ b/guide/guide/.js/src/main/assets/pages/rest.md @@ -705,7 +705,7 @@ Ultimately, if you don't want to use `Future`s, you may replace it with some oth e.g. Monix Task or some IO monad. See [supporting result containers other than `Future`](#supporting-result-containers-other-than-future). -See [streaming serialization workflow](#streaming-serialization-workflow) for details on `monix.reactive.Observable` +See [streaming serialization workflow](#streaming-serialization-workflow) for details on `monix.reactive.Observable` support in streaming REST API methods. ### Customizing serialization @@ -949,7 +949,7 @@ computation which yields a `RestResponse` when run. ### Implementing a server -An existing implementation of REST API trait can be easily turned into a function using +An existing implementation of REST API trait can be easily turned into a function using `RawRest.asHandleRequest` or `RawRest.asHandleRequestWithStreaming` (for server-side streaming support). Therefore, the only thing you need to do to expose your REST API trait as an actual web service it to turn @@ -1096,7 +1096,7 @@ However, this conversion loses the streaming benefits, so it's best used only wh ### Error Handling -Streaming endpoints handle errors similarly to regular endpoints. When an error occurs during streaming: +Streaming endpoints handle errors similarly to regular endpoints. ```scala // Server side @@ -1116,6 +1116,7 @@ client.streamItems("") ``` This allows graceful handling of errors that might occur during streaming operations. +When an error occurs during streaming from the server side, the HTTP connection is closed immediately. This means any in-progress streaming is cut off at the point of error. ### Advanced Streaming Patterns diff --git a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala index b0701e05c..97d27cdde 100644 --- a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala +++ b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala @@ -117,7 +117,6 @@ class RestServlet( private def writeNonEmptyStreamedBody( response: HttpServletResponse, - stream: StreamedRestResponse, body: StreamedBody.NonEmpty, ): Task[Unit] = Task.defer { // The Content-Length header is intentionally omitted for streams. @@ -160,7 +159,11 @@ class RestServlet( .map(_ => response.getOutputStream.write("]".getBytes(jsonList.charset))) } }.onErrorHandle { e => - logger.error(e.getMessage) + // When an error occurs during streaming, we immediately close the connection rather than + // attempting to send an error response. This is intentional because: + // The client has likely already received and started processing partial data + // for structured formats (like JSON arrays), the stream is now in an invalid state + logger.error("Failure during streaming REST response", e) response.getOutputStream.close() } @@ -174,7 +177,7 @@ class RestServlet( case stream: StreamedRestResponse => stream.body match { case StreamedBody.Empty => Task.unit - case neBody: StreamedBody.NonEmpty => writeNonEmptyStreamedBody(response, stream, neBody) + case neBody: StreamedBody.NonEmpty => writeNonEmptyStreamedBody(response, neBody) } } diff --git a/rest/src/main/scala/io/udash/rest/RestDataCompanion.scala b/rest/src/main/scala/io/udash/rest/RestDataCompanion.scala index dde169e81..291d52204 100644 --- a/rest/src/main/scala/io/udash/rest/RestDataCompanion.scala +++ b/rest/src/main/scala/io/udash/rest/RestDataCompanion.scala @@ -5,9 +5,9 @@ import com.avsystem.commons.meta.MacroInstances import com.avsystem.commons.misc.{AbstractValueEnumCompanion, ValueEnum, ValueOf} import com.avsystem.commons.rpc.{AsRaw, AsReal} import com.avsystem.commons.serialization.{GenCodec, TransparentWrapperCompanion} -import io.udash.rest.openapi._ +import io.udash.rest.openapi.* import io.udash.rest.openapi.RestStructure.NameAndAdjusters -import io.udash.rest.raw.{HttpBody, JsonValue, PlainValue, RestResponse} +import io.udash.rest.raw.{HttpBody, JsonValue, PlainValue, RestResponse, StreamedBody, StreamedRestResponse} trait CodecWithStructure[T] { def codec: GenCodec[T] @@ -94,6 +94,18 @@ abstract class RestDataWrapperCompanion[Wrapped, T](implicit implicit def responseAsReal(implicit wrappedAsRaw: AsReal[RestResponse, Wrapped]): AsReal[RestResponse, T] = AsReal.fromTransparentWrapping + implicit def streamedBodyAsRaw(implicit wrappedAsRaw: AsRaw[StreamedBody, Wrapped]): AsRaw[StreamedBody, T] = + AsRaw.fromTransparentWrapping + + implicit def streamedBodyAsReal(implicit wrappedAsRaw: AsReal[StreamedBody, Wrapped]): AsReal[StreamedBody, T] = + AsReal.fromTransparentWrapping + + implicit def streamedResponseAsRaw(implicit wrappedAsRaw: AsRaw[StreamedRestResponse, Wrapped]): AsRaw[StreamedRestResponse, T] = + AsRaw.fromTransparentWrapping + + implicit def streamedResponseAsReal(implicit wrappedAsRaw: AsReal[StreamedRestResponse, Wrapped]): AsReal[StreamedRestResponse, T] = + AsReal.fromTransparentWrapping + implicit def restSchema(implicit wrappedSchema: RestSchema[Wrapped]): RestSchema[T] = nameAndAdjusters.restSchema(wrappedSchema) From 6423bb3f5546009f03e8b6c8afadff953279bac7 Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Wed, 23 Apr 2025 15:56:53 +0200 Subject: [PATCH 091/162] Logging in servlet for client-side stream cancellation --- build.sbt | 2 +- .../scala/io/udash/rest/RestServlet.scala | 66 +++++++++++-------- 2 files changed, 41 insertions(+), 27 deletions(-) diff --git a/build.sbt b/build.sbt index f37c45549..758c5026b 100644 --- a/build.sbt +++ b/build.sbt @@ -9,7 +9,7 @@ name := "udash" Global / excludeLintKeys ++= Set(ideOutputDirectory, ideSkipProject) inThisBuild(Seq( - version := "0.9.0-SNAPSHOT", + version := "0.18.0-SNAPSHOT", organization := "io.udash", resolvers += Resolver.defaultLocal, )) diff --git a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala index 97d27cdde..712cfec7b 100644 --- a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala +++ b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala @@ -3,15 +3,16 @@ package rest import com.avsystem.commons.* import com.avsystem.commons.annotation.explicitGenerics -import com.typesafe.scalalogging.LazyLogging +import com.typesafe.scalalogging.{LazyLogging, Logger as ScalaLogger} import io.udash.rest.RestServlet.* import io.udash.rest.raw.* import io.udash.utils.URLEncoder import monix.eval.Task import monix.execution.Scheduler import monix.reactive.{Consumer, Observable} +import org.slf4j.{Logger, LoggerFactory} -import java.io.ByteArrayOutputStream +import java.io.{ByteArrayOutputStream, EOFException} import java.util.concurrent.atomic.AtomicBoolean import javax.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} import javax.servlet.{AsyncEvent, AsyncListener} @@ -55,12 +56,16 @@ class RestServlet( handleTimeout: FiniteDuration = DefaultHandleTimeout, maxPayloadSize: Long = DefaultMaxPayloadSize, defaultStreamingBatchSize: Int = DefaultStreamingBatchSize, + customLogger: OptArg[Logger] = OptArg.Empty, )(implicit scheduler: Scheduler ) extends HttpServlet with LazyLogging { import RestServlet.* + override protected lazy val logger: ScalaLogger = + ScalaLogger(customLogger.getOrElse(LoggerFactory.getLogger(getClass.getName))) + override def service(request: HttpServletRequest, response: HttpServletResponse): Unit = { val asyncContext = request.startAsync() val completed = new AtomicBoolean(false) @@ -76,25 +81,28 @@ class RestServlet( // readRequest must execute in Jetty thread but we want exceptions to be handled uniformly, hence the Try val udashRequest = Try(readRequest(request)) - val cancelable = Task.defer(handleRequest(udashRequest.get)).flatMap { rr => - Task(setResponseHeaders(response, rr.code, rr.headers)) >> - writeResponseBody(response, rr) - }.executeAsync.runAsync { - case Right(_) => - asyncContext.complete() - case Left(e: HttpErrorException) => - completeWith(writeResponse(response, e.toResponse)) - case Left(e) => - logger.error("Failed to handle REST request", e) - completeWith(writeFailure(response, e.getMessage.opt)) - } + val cancelable = + (for { + restRequest <- Task.fromTry(udashRequest) + restResponse <- handleRequest(restRequest) + _ <- Task(setResponseHeaders(response, restResponse.code, restResponse.headers)) + _ <- writeResponseBody(response, restResponse) + } yield ()).executeAsync.runAsync { + case Right(_) => + asyncContext.complete() + case Left(e: HttpErrorException) => + completeWith(writeResponse(response, e.toResponse)) + case Left(e) => + logger.error("Failed to handle REST request", e) + completeWith(writeFailure(response, e.getMessage.opt)) + } asyncContext.setTimeout(handleTimeout.toMillis) asyncContext.addListener(new AsyncListener { def onComplete(event: AsyncEvent): Unit = () def onTimeout(event: AsyncEvent): Unit = { cancelable.cancel() - completeWith(writeFailure(response, Opt(s"server operation timed out after $handleTimeout"))) + completeWith(writeFailure(response, s"server operation timed out after $handleTimeout".opt)) } def onError(event: AsyncEvent): Unit = () def onStartAsync(event: AsyncEvent): Unit = () @@ -117,13 +125,13 @@ class RestServlet( private def writeNonEmptyStreamedBody( response: HttpServletResponse, - body: StreamedBody.NonEmpty, + responseBody: StreamedBody.NonEmpty, ): Task[Unit] = Task.defer { // The Content-Length header is intentionally omitted for streams. // This signals to the client that the response body size is not predetermined and will be streamed. // Clients implementing the streaming part of the REST interface contract MUST be prepared // to handle responses without Content-Length by reading data incrementally until the stream completes. - body match { + responseBody match { case single: StreamedBody.Single => Task.eval(writeNonEmptyBody(response, single.body)) case binary: StreamedBody.RawBinary => @@ -158,17 +166,23 @@ class RestServlet( }) .map(_ => response.getOutputStream.write("]".getBytes(jsonList.charset))) } - }.onErrorHandle { e => - // When an error occurs during streaming, we immediately close the connection rather than - // attempting to send an error response. This is intentional because: - // The client has likely already received and started processing partial data - // for structured formats (like JSON arrays), the stream is now in an invalid state - logger.error("Failure during streaming REST response", e) - response.getOutputStream.close() + }.onErrorHandle { + case _: EOFException => + logger.warn("Request was cancelled by the client during streaming REST response") + case ex => + // When an error occurs during streaming, we immediately close the connection rather than + // attempting to send an error response. This is intentional because: + // The client has likely already received and started processing partial data + // for structured formats (like JSON arrays), the stream is now in an invalid state + logger.error("Failure during streaming REST response", ex) + response.getOutputStream.close() } - private def writeResponseBody(response: HttpServletResponse, rr: AbstractRestResponse): Task[Unit] = - rr match { + private def writeResponseBody( + response: HttpServletResponse, + restResponse: AbstractRestResponse, + ): Task[Unit] = + restResponse match { case resp: RestResponse => resp.body match { case HttpBody.Empty => Task.unit From facbbb05dfefa5620277cff8710b463547a328b5 Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Thu, 24 Apr 2025 14:38:49 +0200 Subject: [PATCH 092/162] Binary compatibility fixes --- .../scala/io/udash/rest/RestServlet.scala | 15 +++++++- .../io/udash/rest/raw/RestMetadata.scala | 35 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala index 712cfec7b..2e8e96d52 100644 --- a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala +++ b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala @@ -2,7 +2,7 @@ package io.udash package rest import com.avsystem.commons.* -import com.avsystem.commons.annotation.explicitGenerics +import com.avsystem.commons.annotation.{bincompat, explicitGenerics} import com.typesafe.scalalogging.{LazyLogging, Logger as ScalaLogger} import io.udash.rest.RestServlet.* import io.udash.rest.raw.* @@ -49,6 +49,19 @@ object RestServlet { maxPayloadSize = maxPayloadSize, defaultStreamingBatchSize = defaultStreamingBatchSize, ) + + @bincompat private[rest] def apply[RestApi: RawRest.AsRawRpc : RestMetadata]( + apiImpl: RestApi, + handleTimeout: FiniteDuration , + maxPayloadSize: Long, + )(implicit + scheduler: Scheduler + ): RestServlet = apply[RestApi]( + apiImpl, + handleTimeout = handleTimeout, + maxPayloadSize = maxPayloadSize, + defaultStreamingBatchSize = DefaultStreamingBatchSize, + ) } class RestServlet( diff --git a/rest/src/main/scala/io/udash/rest/raw/RestMetadata.scala b/rest/src/main/scala/io/udash/rest/raw/RestMetadata.scala index 869e910fd..9af14514e 100644 --- a/rest/src/main/scala/io/udash/rest/raw/RestMetadata.scala +++ b/rest/src/main/scala/io/udash/rest/raw/RestMetadata.scala @@ -3,6 +3,7 @@ package rest package raw import com.avsystem.commons.* +import com.avsystem.commons.annotation.bincompat import com.avsystem.commons.meta.* import com.avsystem.commons.rpc.* import io.udash.macros.RestMacros @@ -292,6 +293,16 @@ final case class PrefixMetadata[T]( @multi @reifyAnnot streamedResponseAdjusters: List[StreamedResponseAdjuster], @infer @checked result: RestMetadata.Lazy[T], ) extends RestMethodMetadata[T] { + + @bincompat private[rest] def this( + name: String, + methodTag: Prefix, + parametersMetadata: RestParametersMetadata, + requestAdjusters: List[RequestAdjuster], + responseAdjusters: List[ResponseAdjuster], + result: RestMetadata.Lazy[T], + ) = this(name, methodTag, parametersMetadata, requestAdjusters, responseAdjusters, Nil, result) + def methodPath: List[PlainValue] = PlainValue.decodePath(methodTag.path) } @@ -307,6 +318,30 @@ final case class HttpMethodMetadata[T]( @multi @reifyAnnot streamedResponseAdjusters: List[StreamedResponseAdjuster], @infer @checked responseType: HttpResponseType[T], ) extends RestMethodMetadata[T] { + + @bincompat private[rest] def this( + name: String, + methodTag: HttpMethodTag, + bodyTypeTag: BodyTypeTag, + parametersMetadata: RestParametersMetadata, + bodyParams: List[ParamMetadata[_]], + formBody: Boolean, + requestAdjusters: List[RequestAdjuster], + responseAdjusters: List[ResponseAdjuster], + responseType: HttpResponseType[T], + ) = this( + name, + methodTag, + bodyTypeTag, + parametersMetadata, + bodyParams, + formBody, + requestAdjusters, + responseAdjusters, + Nil, + responseType + ) + val method: HttpMethod = methodTag.method val customBody: Boolean = bodyTypeTag match { From 77ecfe0e1de6c298c1657d643efd22dc854c02ec Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Fri, 25 Apr 2025 10:48:25 +0200 Subject: [PATCH 093/162] Allow different content types for StreamedBody.RawBinary --- .../io/udash/rest/jetty/JettyRestClient.scala | 13 ++++++++----- .../io/udash/rest/raw/StreamedBody.scala | 19 +++++++++++-------- .../scala/io/udash/rest/raw/RawRestTest.scala | 4 ++-- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala index 44be2ebce..8ab41999d 100644 --- a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala +++ b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala @@ -89,11 +89,9 @@ final class JettyRestClient( if (contentLength == -1) { val contentTypeOpt = response.getHeaders.get(HttpHeader.CONTENT_TYPE).opt val mediaTypeOpt = contentTypeOpt.map(MimeTypes.getContentTypeWithoutCharset) - val bodyOpt = mediaTypeOpt matchOpt { - case Opt(HttpBody.OctetStreamType) => - StreamedBody.RawBinary(content = rawContentSubject.doOnSubscriptionCancel(cancelRequest)) - case Opt(HttpBody.JsonType) => - val charset = contentTypeOpt.map(MimeTypes.getCharsetFromContentType).getOrElse(HttpBody.Utf8Charset) + val charsetOpt = contentTypeOpt.map(MimeTypes.getCharsetFromContentType) + val bodyOpt = (mediaTypeOpt, charsetOpt) matchOpt { + case (Opt(HttpBody.JsonType), Opt(charset)) => // suboptimal - maybe "online" parsing is possible using Jackson / other lib without waiting for full content ? StreamedBody.JsonList( elements = Observable @@ -109,6 +107,11 @@ final class JettyRestClient( .onErrorFallbackTo(Observable.raiseError(JettyRestClient.Streaming)), charset = charset, ) + case (Opt(mediaType), _) => + StreamedBody.binary( + content = rawContentSubject.doOnSubscriptionCancel(cancelRequest), + contentType = contentTypeOpt.getOrElse(mediaType), + ) } bodyOpt.mapOr( { diff --git a/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala b/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala index d8891dbc9..aefa15aa0 100644 --- a/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala +++ b/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala @@ -35,11 +35,10 @@ object StreamedBody extends StreamedBodyLowPrio { * The content is delivered as a stream of byte arrays which can be processed incrementally. * Useful for large binary files or content that is generated dynamically. */ - final case class RawBinary(content: Observable[Array[Byte]]) extends NonEmpty { - val contentType: String = HttpBody.OctetStreamType - - override def toString: String = super.toString - } + final case class RawBinary( + content: Observable[Array[Byte]], + override val contentType: String, + ) extends NonEmpty /** * Represents a streamed list of JSON values. @@ -52,8 +51,6 @@ object StreamedBody extends StreamedBodyLowPrio { customBatchSize: Opt[Int] = Opt.Empty, ) extends NonEmpty { val contentType: String = s"${HttpBody.JsonType};charset=$charset" - - override def toString: String = super.toString } /** @@ -67,6 +64,12 @@ object StreamedBody extends StreamedBodyLowPrio { def empty: StreamedBody = Empty + def binary( + content: Observable[Array[Byte]], + contentType: String = HttpBody.OctetStreamType, + ): StreamedBody = + RawBinary(content, contentType) + def fromHttpBody(body: HttpBody): StreamedBody = body match { case HttpBody.Empty => StreamedBody.Empty case nonEmpty: HttpBody.NonEmpty => StreamedBody.Single(nonEmpty) @@ -89,7 +92,7 @@ object StreamedBody extends StreamedBodyLowPrio { implicit val rawBinaryBodyForByteArray: AsRawReal[StreamedBody, Observable[Array[Byte]]] = AsRawReal.create( - bytes => RawBinary(bytes), + bytes => StreamedBody.binary(bytes), body => StreamedBody.castOrFail[RawBinary](body).content, ) } diff --git a/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala b/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala index 5e57d8866..16e71ee79 100644 --- a/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala +++ b/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala @@ -126,9 +126,9 @@ class RawRestTest extends AnyFunSuite with ScalaFutures with Matchers { elements.toListL.map { list => s" application/json;charset=$charset\n[${list.map(_.value).mkString(",")}]" } - case StreamedBody.RawBinary(content) => + case StreamedBody.RawBinary(content, tpe) => content.toListL.map { list => - s" application/octet-stream\n${list.flatMap(_.iterator.map(b => f"$b%02X")).mkString}" + s" $tpe\n${list.flatMap(_.iterator.map(b => f"$b%02X")).mkString}" } case StreamedBody.Single(body) => Task.now(repr(body, inNewLine = false)) From 6113dba2c3e889effaa25447dd064279267c2c70 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Fri, 2 May 2025 20:17:21 +0000 Subject: [PATCH 094/162] Update jetty-client to 12.0.20 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6af3f39a3..b81c0db12 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -28,7 +28,7 @@ object Dependencies { val scalaLoggingVersion = "3.9.5" - val jettyVersion = "12.0.19" + val jettyVersion = "12.0.20" val typesafeConfigVersion = "1.4.3" val flexmarkVersion = "0.64.8" val logbackVersion = "1.3.15" From 86e6699e2eab27831e8187a8b2bf05aa92c92796 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sun, 4 May 2025 17:40:51 +0000 Subject: [PATCH 095/162] Update selenium-java to 4.32.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6af3f39a3..8a78a6839 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -42,7 +42,7 @@ object Dependencies { val bootstrap4DatepickerVersion = "5.39.0" val momentJsVersion = "2.30.1" - val seleniumVersion = "4.31.0" + val seleniumVersion = "4.32.0" val webDriverManagerVersion = "6.1.0" val scalaJsBenchmarkVersion = "0.10.0" From 37f403875608246d542e277dbebc9660b02010cc Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Fri, 9 May 2025 11:59:20 +0200 Subject: [PATCH 096/162] scala-commons 2.22.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index c50516626..967ef110c 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -14,7 +14,7 @@ object Dependencies { val scalaCssVersion = "1.0.0" val servletVersion = "4.0.1" - val avsCommonsVersion = "2.21.0" + val avsCommonsVersion = "2.22.0" val atmosphereJSVersion = "3.1.3" val atmosphereVersion = "2.7.15" From e18fd5c279b9178ec64b4d58e6d8b4e55e8a36e2 Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Tue, 13 May 2025 10:25:40 +0200 Subject: [PATCH 097/162] Switch credential host to Sonatype Central --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index f37c45549..2b8a021fe 100644 --- a/build.sbt +++ b/build.sbt @@ -30,6 +30,7 @@ val deploymentConfiguration = Seq( pomIncludeRepository := { _ => false }, publishTo := sonatypePublishToBundle.value, + sonatypeCredentialHost := Sonatype.sonatypeCentralHost, credentials in Global += Credentials( "Sonatype Nexus Repository Manager", From a1d59dcc3811e9228dabef747f934100be459dde Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Tue, 13 May 2025 11:00:56 +0200 Subject: [PATCH 098/162] Switch to sbt-ci-release --- build.sbt | 36 ++++++++---------------------------- project/plugins.sbt | 5 +---- 2 files changed, 9 insertions(+), 32 deletions(-) diff --git a/build.sbt b/build.sbt index 2b8a021fe..b844cceb2 100644 --- a/build.sbt +++ b/build.sbt @@ -9,7 +9,6 @@ name := "udash" Global / excludeLintKeys ++= Set(ideOutputDirectory, ideSkipProject) inThisBuild(Seq( - version := "0.9.0-SNAPSHOT", organization := "io.udash", resolvers += Resolver.defaultLocal, )) @@ -25,36 +24,17 @@ val browserCapabilities: Capabilities = { // Deployment configuration val deploymentConfiguration = Seq( - publishMavenStyle := true, - Test / publishArtifact := false, pomIncludeRepository := { _ => false }, - - publishTo := sonatypePublishToBundle.value, sonatypeCredentialHost := Sonatype.sonatypeCentralHost, - - credentials in Global += Credentials( - "Sonatype Nexus Repository Manager", - "oss.sonatype.org", - sys.env.getOrElse("SONATYPE_USERNAME", ""), - sys.env.getOrElse("SONATYPE_PASSWORD", "") - ), - licenses := Seq(License.Apache2), - - pomExtra := { - https://github.com/UdashFramework/udash-core - - git@github.com:UdashFramework/udash-core.git - scm:git@github.com:UdashFramework/udash-core.git - - - - avsystem - AVSystem - http://www.avsystem.com/ - - - } + scmInfo := Some(ScmInfo( + browseUrl = url("https://github.com/UdashFramework/udash-core"), + connection = "scm:git:git@github.com:UdashFramework/udash-core.git", + devConnection = Some("scm:git:git@github.com:UdashFramework/udash-core.git"), + )), + developers := List( + Developer("ddworak", "Dawid Dworak", "d.dworak@avsystem.com", url("https://github.com/ddworak")), + ), ) val commonSettings = Seq( diff --git a/project/plugins.sbt b/project/plugins.sbt index 22e3a457b..32d66d6de 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -9,7 +9,4 @@ addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.4") addSbtPlugin("org.jetbrains.scala" % "sbt-ide-settings" % "1.1.2") addSbtPlugin("com.github.sbt" % "sbt-less" % "2.0.1") addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.1") - -// Deployment configuration -addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.1") -addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.12.2") \ No newline at end of file +addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.9.3") From 24fda727d230929391c4350cf7fc8c815947c9e8 Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Tue, 13 May 2025 11:13:45 +0200 Subject: [PATCH 099/162] Fix publishing pipeline --- .github/workflows/ci.yml | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 534e2cea8..7173adfc6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,20 +52,10 @@ jobs: - name: Get version id: get_tag_name run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - - name: Import GPG key - env: - #exported via `gpg -a --export-secret-keys | cat -e | sed 's/\$/\\n/g' | xclip -selection clipboard` and added to org secrets - SONATYPE_GPG: ${{ secrets.SONATYPE_GPG }} - run: echo -e $SONATYPE_GPG | gpg --import - - name: Publish artifacts env: - SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} + PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} + PGP_SECRET: ${{ secrets.PGP_SECRET }} SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} - run: sbt 'set ThisBuild/version := "${{ steps.get_tag_name.outputs.VERSION }}"' +publishSigned - - name: Release Sonatype bundle - #https://github.com/xerial/sbt-sonatype#publishing-your-artifact - if: ${{ !endsWith(steps.get_tag_name.outputs.VERSION, 'SNAPSHOT') }} - env: SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} - SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} - run: sbt 'set ThisBuild/version := "${{ steps.get_tag_name.outputs.VERSION }}"' sonatypeBundleRelease + run: sbt ci-release From 48c8e17a530ed705475cc33ffc2f4dc9b4751ee1 Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Tue, 13 May 2025 11:37:19 +0200 Subject: [PATCH 100/162] Move Sonatype Central setting to top-level build settings --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index b844cceb2..ac4ff7523 100644 --- a/build.sbt +++ b/build.sbt @@ -11,6 +11,7 @@ Global / excludeLintKeys ++= Set(ideOutputDirectory, ideSkipProject) inThisBuild(Seq( organization := "io.udash", resolvers += Resolver.defaultLocal, + sonatypeCredentialHost := Sonatype.sonatypeCentralHost, )) val forIdeaImport = System.getProperty("idea.managed", "false").toBoolean && System.getProperty("idea.runid") == null @@ -25,7 +26,6 @@ val browserCapabilities: Capabilities = { // Deployment configuration val deploymentConfiguration = Seq( pomIncludeRepository := { _ => false }, - sonatypeCredentialHost := Sonatype.sonatypeCentralHost, licenses := Seq(License.Apache2), scmInfo := Some(ScmInfo( browseUrl = url("https://github.com/UdashFramework/udash-core"), From ff0bc97fadcad02b4435a48fd2e3a56e0b711750 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 13 May 2025 09:43:00 +0000 Subject: [PATCH 101/162] Update jetty-client to 12.0.21 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 967ef110c..facf3ebda 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -28,7 +28,7 @@ object Dependencies { val scalaLoggingVersion = "3.9.5" - val jettyVersion = "12.0.20" + val jettyVersion = "12.0.21" val typesafeConfigVersion = "1.4.3" val flexmarkVersion = "0.64.8" val logbackVersion = "1.3.15" From 567645944a157d4f848313cf7d44b25a897b75b0 Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Tue, 13 May 2025 12:14:43 +0200 Subject: [PATCH 102/162] Add missing project URL for Central publishing --- build.sbt | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/build.sbt b/build.sbt index ac4ff7523..9d8e58b47 100644 --- a/build.sbt +++ b/build.sbt @@ -12,6 +12,16 @@ inThisBuild(Seq( organization := "io.udash", resolvers += Resolver.defaultLocal, sonatypeCredentialHost := Sonatype.sonatypeCentralHost, + homepage := Some(url("https://udash.io")), + licenses := Seq(License.Apache2), + scmInfo := Some(ScmInfo( + browseUrl = url("https://github.com/UdashFramework/udash-core"), + connection = "scm:git:git@github.com:UdashFramework/udash-core.git", + devConnection = Some("scm:git:git@github.com:UdashFramework/udash-core.git"), + )), + developers := List( + Developer("ddworak", "Dawid Dworak", "d.dworak@avsystem.com", url("https://github.com/ddworak")), + ), )) val forIdeaImport = System.getProperty("idea.managed", "false").toBoolean && System.getProperty("idea.runid") == null @@ -23,20 +33,6 @@ val browserCapabilities: Capabilities = { new FirefoxOptions().setHeadless(true).setLogLevel(FirefoxDriverLogLevel.WARN) } -// Deployment configuration -val deploymentConfiguration = Seq( - pomIncludeRepository := { _ => false }, - licenses := Seq(License.Apache2), - scmInfo := Some(ScmInfo( - browseUrl = url("https://github.com/UdashFramework/udash-core"), - connection = "scm:git:git@github.com:UdashFramework/udash-core.git", - devConnection = Some("scm:git:git@github.com:UdashFramework/udash-core.git"), - )), - developers := List( - Developer("ddworak", "Dawid Dworak", "d.dworak@avsystem.com", url("https://github.com/ddworak")), - ), -) - val commonSettings = Seq( scalaVersion := Dependencies.versionOfScala, crossScalaVersions := Seq(Dependencies.versionOfScala), @@ -66,7 +62,8 @@ val commonSettings = Seq( Test / ideOutputDirectory := Some(target.value.getParentFile / "out/test"), libraryDependencies ++= Dependencies.compilerPlugins.value, libraryDependencies ++= Dependencies.commonTestDeps.value, -) ++ deploymentConfiguration + pomIncludeRepository := { _ => false }, +) val commonJsSettings = commonSettings ++ Seq( Test / scalaJSStage := FastOptStage, From d3536a85b637861c3c15e2128548e9f0b68ca63f Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sun, 18 May 2025 18:41:53 +0000 Subject: [PATCH 103/162] Update upickle to 4.2.1 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index facf3ebda..41f56bb7d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -19,7 +19,7 @@ object Dependencies { val atmosphereJSVersion = "3.1.3" val atmosphereVersion = "2.7.15" - val upickleVersion = "4.1.0" // Tests only + val upickleVersion = "4.2.1" // Tests only val circeVersion = "0.14.13" // Tests only val circeDerivationVersion = "0.13.0-M5" // Tests only val monixVersion = "3.4.1" // udash-rest only From 2545cc12a2997532c98da51f793f83e6742297c3 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 24 May 2025 21:38:35 +0000 Subject: [PATCH 104/162] Update selenium-java to 4.33.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index facf3ebda..f02945031 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -42,7 +42,7 @@ object Dependencies { val bootstrap4DatepickerVersion = "5.39.0" val momentJsVersion = "2.30.1" - val seleniumVersion = "4.32.0" + val seleniumVersion = "4.33.0" val webDriverManagerVersion = "6.1.0" val scalaJsBenchmarkVersion = "0.10.0" From 091c3e36ac09dad7d8ff9f09d73b29d7a286704a Mon Sep 17 00:00:00 2001 From: ojsyrifsdj Date: Wed, 28 May 2025 10:36:08 +0200 Subject: [PATCH 105/162] Rest benchmarks --- .../main/scala/io/udash/rest/RestApi.scala | 58 ++++++++++++ .../io/udash/rest/StreamingRestApi.scala | 94 +++++++++++++++++++ build.sbt | 11 ++- project/plugins.sbt | 3 +- 4 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 benchmarks/src/main/scala/io/udash/rest/RestApi.scala create mode 100644 benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala diff --git a/benchmarks/src/main/scala/io/udash/rest/RestApi.scala b/benchmarks/src/main/scala/io/udash/rest/RestApi.scala new file mode 100644 index 000000000..69e5b9c7a --- /dev/null +++ b/benchmarks/src/main/scala/io/udash/rest/RestApi.scala @@ -0,0 +1,58 @@ +package io.udash.rest + +import io.udash.rest.raw.RawRest +import monix.eval.Task +import monix.execution.Scheduler +import org.openjdk.jmh.annotations.* + +import java.util.concurrent.TimeUnit +import scala.concurrent.Await +import scala.concurrent.duration.Duration + +private object RestApi { + trait RestTestApi { + @GET def simpleNumbers(size: Int): Task[List[Int]] + } + + object RestTestApi extends DefaultRestApiCompanion[RestTestApi] { + final class Impl extends RestTestApi { + + def simpleNumbers(size: Int): Task[List[Int]] = + Task.eval(Range(0, size).toList) + } + } + + private def creteApiProxy(): RestTestApi = { + val apiImpl = new RestTestApi.Impl() + val handler = RawRest.asHandleRequest[RestTestApi](apiImpl) + RawRest.fromHandleRequest[RestTestApi](handler) + } +} + + +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@BenchmarkMode(Array(Mode.Throughput)) +@State(Scope.Thread) +class RestApi { + implicit def scheduler: Scheduler = Scheduler.global + private final val proxy = RestApi.creteApiProxy() + + @Benchmark + def smallNumbersArray(): Unit = { + waitEndpoint(10) + } + + @Benchmark + def mediumNumbersArray(): Unit = { + waitEndpoint(200) + } + + @Benchmark + def hugeNumbersArray(): Unit = { + waitEndpoint(5000) + } + + private def waitEndpoint(samples: Int): Unit = { + Await.result(this.proxy.simpleNumbers(samples).runToFuture, Duration.apply(10, TimeUnit.SECONDS)) + } +} \ No newline at end of file diff --git a/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala b/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala new file mode 100644 index 000000000..343f0510a --- /dev/null +++ b/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala @@ -0,0 +1,94 @@ +package io.udash.rest + +import io.udash.rest.raw.{RawRest, RestRequest, RestResponse, StreamedRestResponse} +import monix.eval.Task +import monix.execution.Scheduler +import monix.reactive.Observable +import org.openjdk.jmh.annotations.* + +import java.util.concurrent.TimeUnit +import scala.concurrent.Await +import scala.concurrent.duration.Duration + +private object StreamingRestApi { + trait RestTestApi { + @GET def simpleNumbers(size: Int): Observable[Int] + @GET def simpleNumbersWithoutStreaming(size: Int): Task[List[Int]] + } + + object RestTestApi extends DefaultRestApiCompanion[RestTestApi] { + final class Impl extends RestTestApi { + + def simpleNumbers(size: Int): Observable[Int] = + Observable.fromIterable(Range(0, size)) + + def simpleNumbersWithoutStreaming(size: Int): Task[List[Int]] = + Task.eval(Range(0, size).toList) + } + } + + private def creteApiProxy(): RestTestApi = { + val apiImpl = new RestTestApi.Impl() + val streamingServerHandle = RawRest.asHandleRequestWithStreaming[RestTestApi](apiImpl) + val streamingClientHandler = new RawRest.RestRequestHandler { + override def handleRequest(request: RestRequest): Task[RestResponse] = + streamingServerHandle(request).map(_.asInstanceOf[RestResponse]) + + override def handleRequestStream(request: RestRequest): Task[StreamedRestResponse] = + streamingServerHandle(request).map(_.asInstanceOf[StreamedRestResponse]) + } + RawRest.fromHandleRequestWithStreaming[RestTestApi](streamingClientHandler) + } +} + + +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@BenchmarkMode(Array(Mode.Throughput)) +@State(Scope.Thread) +class StreamingRestApi { + implicit def scheduler: Scheduler = Scheduler.global + private final val proxy = StreamingRestApi.creteApiProxy() + + + @Benchmark + def smallNumbersArray(): Unit = { + waitStreamingEndpoint(10) + } + + @Benchmark + def mediumNumbersArray(): Unit = { + waitStreamingEndpoint(200) + } + + @Benchmark + def hugeNumbersArray(): Unit = { + waitStreamingEndpoint(5000) + } + + @Benchmark + def smallNumbersArrayWithoutStreaming(): Unit = { + waitEndpointWithoutStreaming(10) + } + + @Benchmark + def mediumNumbersArrayWithoutStreaming(): Unit = { + waitEndpointWithoutStreaming(200) + } + + @Benchmark + def hugeNumbersArrayWithoutStreaming(): Unit = { + waitEndpointWithoutStreaming(5000) + } + + private def waitEndpointWithoutStreaming(samples: Int): Unit = { + wait(this.proxy.simpleNumbersWithoutStreaming(samples)) + } + + private def waitStreamingEndpoint(samples: Int): Unit = { + wait(this.proxy.simpleNumbers(samples).toListL) + } + + private def wait[T](task: Task[List[T]]): Unit = { + Await.result(task.runToFuture, Duration.apply(10, TimeUnit.SECONDS)) + } +} \ No newline at end of file diff --git a/build.sbt b/build.sbt index 758c5026b..ca43ca718 100644 --- a/build.sbt +++ b/build.sbt @@ -3,6 +3,7 @@ import org.openqa.selenium.firefox.{FirefoxDriverLogLevel, FirefoxOptions} import org.scalajs.jsdependencies.sbtplugin.JSModuleID import org.scalajs.jsenv.jsdomnodejs.JSDOMNodeJSEnv import org.scalajs.jsenv.selenium.SeleniumJSEnv +import pl.project13.scala.sbt.JmhPlugin name := "udash" @@ -483,4 +484,12 @@ lazy val `guide-selenium` = `guide-homepage` / Compile / compileStatics, `guide-guide` / Compile / compileStatics, ).value - ) \ No newline at end of file + ) + +lazy val benchmarksJVM = project.in(file("benchmarks")) + .enablePlugins(JmhPlugin) + .dependsOn(jvmLibraries.map(p => p: ClasspathDep[ProjectReference]): _*) + .settings( + commonSettings, + noPublishSettings, + ) \ No newline at end of file diff --git a/project/plugins.sbt b/project/plugins.sbt index 07fe5c046..e0ac342d8 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -12,4 +12,5 @@ addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.1") // Deployment configuration addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.1") -addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.12.2") \ No newline at end of file +addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.12.2") +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.7") \ No newline at end of file From 3e052e843c7f17771c1b0b94e205774d788b596d Mon Sep 17 00:00:00 2001 From: ojsyrifsdj Date: Wed, 28 May 2025 14:26:29 +0200 Subject: [PATCH 106/162] Rest benchmarks --- .../main/scala/io/udash/rest/RestApi.scala | 16 ++--- .../scala/io/udash/rest/RestExampleData.scala | 24 +++++++ .../io/udash/rest/StreamingRestApi.scala | 71 +++++++++++++++---- 3 files changed, 89 insertions(+), 22 deletions(-) create mode 100644 benchmarks/src/main/scala/io/udash/rest/RestExampleData.scala diff --git a/benchmarks/src/main/scala/io/udash/rest/RestApi.scala b/benchmarks/src/main/scala/io/udash/rest/RestApi.scala index 69e5b9c7a..9947e4445 100644 --- a/benchmarks/src/main/scala/io/udash/rest/RestApi.scala +++ b/benchmarks/src/main/scala/io/udash/rest/RestApi.scala @@ -11,14 +11,14 @@ import scala.concurrent.duration.Duration private object RestApi { trait RestTestApi { - @GET def simpleNumbers(size: Int): Task[List[Int]] + @GET def exampleEndpoint(size: Int): Task[List[RestExampleData]] } object RestTestApi extends DefaultRestApiCompanion[RestTestApi] { final class Impl extends RestTestApi { - def simpleNumbers(size: Int): Task[List[Int]] = - Task.eval(Range(0, size).toList) + def exampleEndpoint(size: Int): Task[List[RestExampleData]] = + RestExampleData.generateRandomList(size) } } @@ -30,7 +30,7 @@ private object RestApi { } -@OutputTimeUnit(TimeUnit.MILLISECONDS) +@OutputTimeUnit(TimeUnit.SECONDS) @BenchmarkMode(Array(Mode.Throughput)) @State(Scope.Thread) class RestApi { @@ -38,21 +38,21 @@ class RestApi { private final val proxy = RestApi.creteApiProxy() @Benchmark - def smallNumbersArray(): Unit = { + def smallArray(): Unit = { waitEndpoint(10) } @Benchmark - def mediumNumbersArray(): Unit = { + def mediumArray(): Unit = { waitEndpoint(200) } @Benchmark - def hugeNumbersArray(): Unit = { + def hugeArray(): Unit = { waitEndpoint(5000) } private def waitEndpoint(samples: Int): Unit = { - Await.result(this.proxy.simpleNumbers(samples).runToFuture, Duration.apply(10, TimeUnit.SECONDS)) + Await.result(this.proxy.exampleEndpoint(samples).runToFuture, Duration.apply(10, TimeUnit.SECONDS)) } } \ No newline at end of file diff --git a/benchmarks/src/main/scala/io/udash/rest/RestExampleData.scala b/benchmarks/src/main/scala/io/udash/rest/RestExampleData.scala new file mode 100644 index 000000000..3b968f7f9 --- /dev/null +++ b/benchmarks/src/main/scala/io/udash/rest/RestExampleData.scala @@ -0,0 +1,24 @@ +package io.udash.rest + +import monix.eval.Task +import monix.reactive.Observable + +import scala.util.Random + +case class RestExampleData(number: Long, string: String) + +object RestExampleData extends RestDataCompanion[RestExampleData]{ + private def random() = { + RestExampleData( + Random.nextLong(), + Iterator.continually(Random.nextPrintableChar()).take(200).mkString + ) + } + + def generateRandomObservable(size: Int): Observable[RestExampleData] = + Observable.fromIterable(Range(0, size).map(_ => RestExampleData.random())) + + def generateRandomList(size: Int): Task[List[RestExampleData]] = + Task.eval(Range(0, size).toList.map(_ => RestExampleData.random())) + +} \ No newline at end of file diff --git a/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala b/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala index 343f0510a..67b69c26b 100644 --- a/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala +++ b/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala @@ -12,18 +12,31 @@ import scala.concurrent.duration.Duration private object StreamingRestApi { trait RestTestApi { - @GET def simpleNumbers(size: Int): Observable[Int] - @GET def simpleNumbersWithoutStreaming(size: Int): Task[List[Int]] + @GET def exampleEndpoint(size: Int): Observable[RestExampleData] + + @streamingResponseBatchSize(10) + @GET def exampleEndpointBatch10(size: Int): Observable[RestExampleData] + + @streamingResponseBatchSize(500) + @GET def exampleEndpointBatch500(size: Int): Observable[RestExampleData] + + @GET def exampleEndpointWithoutStreaming(size: Int): Task[List[RestExampleData]] } object RestTestApi extends DefaultRestApiCompanion[RestTestApi] { final class Impl extends RestTestApi { - def simpleNumbers(size: Int): Observable[Int] = - Observable.fromIterable(Range(0, size)) + def exampleEndpoint(size: Int): Observable[RestExampleData] = + RestExampleData.generateRandomObservable(size) - def simpleNumbersWithoutStreaming(size: Int): Task[List[Int]] = - Task.eval(Range(0, size).toList) + def exampleEndpointBatch10(size: Int): Observable[RestExampleData] = + RestExampleData.generateRandomObservable(size) + + def exampleEndpointBatch500(size: Int): Observable[RestExampleData] = + RestExampleData.generateRandomObservable(size) + + def exampleEndpointWithoutStreaming(size: Int): Task[List[RestExampleData]] = + RestExampleData.generateRandomList(size) } } @@ -51,41 +64,71 @@ class StreamingRestApi { @Benchmark - def smallNumbersArray(): Unit = { + def smallArray(): Unit = { waitStreamingEndpoint(10) } @Benchmark - def mediumNumbersArray(): Unit = { + def mediumArray(): Unit = { waitStreamingEndpoint(200) } @Benchmark - def hugeNumbersArray(): Unit = { + def hugeArray(): Unit = { waitStreamingEndpoint(5000) } @Benchmark - def smallNumbersArrayWithoutStreaming(): Unit = { + def smallArrayBatch10(): Unit = { + wait(this.proxy.exampleEndpointBatch10(10).toListL) + } + + @Benchmark + def mediumArrayBatch10(): Unit = { + wait(this.proxy.exampleEndpointBatch10(200).toListL) + } + + @Benchmark + def hugeArrayBatch10(): Unit = { + wait(this.proxy.exampleEndpointBatch10(5000).toListL) + } + + @Benchmark + def smallArrayBatch500(): Unit = { + wait(this.proxy.exampleEndpointBatch500(10).toListL) + } + + @Benchmark + def mediumArrayBatch500(): Unit = { + wait(this.proxy.exampleEndpointBatch500(200).toListL) + } + + @Benchmark + def hugeArrayBatch500(): Unit = { + wait(this.proxy.exampleEndpointBatch500(5000).toListL) + } + + @Benchmark + def smallArrayWithoutStreaming(): Unit = { waitEndpointWithoutStreaming(10) } @Benchmark - def mediumNumbersArrayWithoutStreaming(): Unit = { + def mediumArrayWithoutStreaming(): Unit = { waitEndpointWithoutStreaming(200) } @Benchmark - def hugeNumbersArrayWithoutStreaming(): Unit = { + def hugeArrayWithoutStreaming(): Unit = { waitEndpointWithoutStreaming(5000) } private def waitEndpointWithoutStreaming(samples: Int): Unit = { - wait(this.proxy.simpleNumbersWithoutStreaming(samples)) + wait(this.proxy.exampleEndpointWithoutStreaming(samples)) } private def waitStreamingEndpoint(samples: Int): Unit = { - wait(this.proxy.simpleNumbers(samples).toListL) + wait(this.proxy.exampleEndpoint(samples).toListL) } private def wait[T](task: Task[List[T]]): Unit = { From cfcb0070622fe3eac208d1b32fd00c4accbbbe9f Mon Sep 17 00:00:00 2001 From: ojsyrifsdj Date: Wed, 28 May 2025 16:12:12 +0200 Subject: [PATCH 107/162] Rest benchmarks - setup --- .../main/scala/io/udash/rest/RestApi.scala | 35 ++++++--- .../scala/io/udash/rest/RestExampleData.scala | 15 ++-- .../io/udash/rest/StreamingRestApi.scala | 71 +++++++++++-------- 3 files changed, 75 insertions(+), 46 deletions(-) diff --git a/benchmarks/src/main/scala/io/udash/rest/RestApi.scala b/benchmarks/src/main/scala/io/udash/rest/RestApi.scala index 9947e4445..250687ca4 100644 --- a/benchmarks/src/main/scala/io/udash/rest/RestApi.scala +++ b/benchmarks/src/main/scala/io/udash/rest/RestApi.scala @@ -1,5 +1,6 @@ package io.udash.rest +import io.udash.rest.RestExampleData.RestResponseSize import io.udash.rest.raw.RawRest import monix.eval.Task import monix.execution.Scheduler @@ -11,21 +12,27 @@ import scala.concurrent.duration.Duration private object RestApi { trait RestTestApi { - @GET def exampleEndpoint(size: Int): Task[List[RestExampleData]] + @GET def exampleEndpoint(size: RestResponseSize): Task[List[RestExampleData]] } object RestTestApi extends DefaultRestApiCompanion[RestTestApi] { final class Impl extends RestTestApi { + private var responses: Map[RestResponseSize, List[RestExampleData]] = + Map.empty - def exampleEndpoint(size: Int): Task[List[RestExampleData]] = - RestExampleData.generateRandomList(size) + def exampleEndpoint(size: RestResponseSize): Task[List[RestExampleData]] = + Task.eval(responses(size)) + + def generateResponses(): Unit = { + this.responses = RestResponseSize.values.map(size => size -> RestExampleData.generateRandomList(size)).toMap + } } } - private def creteApiProxy(): RestTestApi = { + private def creteApiProxy(): (RestTestApi.Impl, RestTestApi) = { val apiImpl = new RestTestApi.Impl() val handler = RawRest.asHandleRequest[RestTestApi](apiImpl) - RawRest.fromHandleRequest[RestTestApi](handler) + (apiImpl, RawRest.fromHandleRequest[RestTestApi](handler)) } } @@ -35,24 +42,30 @@ private object RestApi { @State(Scope.Thread) class RestApi { implicit def scheduler: Scheduler = Scheduler.global - private final val proxy = RestApi.creteApiProxy() + + private final val (impl, proxy) = RestApi.creteApiProxy() + + @Setup(Level.Trial) + def setup(): Unit = { + this.impl.generateResponses() + } @Benchmark def smallArray(): Unit = { - waitEndpoint(10) + waitEndpoint(RestResponseSize.Small) } @Benchmark def mediumArray(): Unit = { - waitEndpoint(200) + waitEndpoint(RestResponseSize.Medium) } @Benchmark def hugeArray(): Unit = { - waitEndpoint(5000) + waitEndpoint(RestResponseSize.Huge) } - private def waitEndpoint(samples: Int): Unit = { - Await.result(this.proxy.exampleEndpoint(samples).runToFuture, Duration.apply(10, TimeUnit.SECONDS)) + private def waitEndpoint(size: RestResponseSize): Unit = { + Await.result(this.proxy.exampleEndpoint(size).runToFuture, Duration.apply(10, TimeUnit.SECONDS)) } } \ No newline at end of file diff --git a/benchmarks/src/main/scala/io/udash/rest/RestExampleData.scala b/benchmarks/src/main/scala/io/udash/rest/RestExampleData.scala index 3b968f7f9..96fa83091 100644 --- a/benchmarks/src/main/scala/io/udash/rest/RestExampleData.scala +++ b/benchmarks/src/main/scala/io/udash/rest/RestExampleData.scala @@ -1,5 +1,6 @@ package io.udash.rest +import com.avsystem.commons.misc.{AbstractValueEnum, EnumCtx, ValueEnum} import monix.eval.Task import monix.reactive.Observable @@ -8,6 +9,13 @@ import scala.util.Random case class RestExampleData(number: Long, string: String) object RestExampleData extends RestDataCompanion[RestExampleData]{ + final case class RestResponseSize(value: Int)(implicit enumCtx: EnumCtx) extends AbstractValueEnum + object RestResponseSize extends RestValueEnumCompanion[RestResponseSize] { + final val Small: Value = new RestResponseSize(10) + final val Medium: Value = new RestResponseSize(200) + final val Huge: Value = new RestResponseSize(5000) + } + private def random() = { RestExampleData( Random.nextLong(), @@ -15,10 +23,7 @@ object RestExampleData extends RestDataCompanion[RestExampleData]{ ) } - def generateRandomObservable(size: Int): Observable[RestExampleData] = - Observable.fromIterable(Range(0, size).map(_ => RestExampleData.random())) - - def generateRandomList(size: Int): Task[List[RestExampleData]] = - Task.eval(Range(0, size).toList.map(_ => RestExampleData.random())) + def generateRandomList(size: RestResponseSize): List[RestExampleData] = + Range(0, size.value).toList.map(_ => RestExampleData.random()) } \ No newline at end of file diff --git a/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala b/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala index 67b69c26b..6ad936941 100644 --- a/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala +++ b/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala @@ -1,5 +1,6 @@ package io.udash.rest +import io.udash.rest.RestExampleData.RestResponseSize import io.udash.rest.raw.{RawRest, RestRequest, RestResponse, StreamedRestResponse} import monix.eval.Task import monix.execution.Scheduler @@ -12,35 +13,41 @@ import scala.concurrent.duration.Duration private object StreamingRestApi { trait RestTestApi { - @GET def exampleEndpoint(size: Int): Observable[RestExampleData] + @GET def exampleEndpoint(size: RestResponseSize): Observable[RestExampleData] @streamingResponseBatchSize(10) - @GET def exampleEndpointBatch10(size: Int): Observable[RestExampleData] + @GET def exampleEndpointBatch10(size: RestResponseSize): Observable[RestExampleData] @streamingResponseBatchSize(500) - @GET def exampleEndpointBatch500(size: Int): Observable[RestExampleData] + @GET def exampleEndpointBatch500(size: RestResponseSize): Observable[RestExampleData] - @GET def exampleEndpointWithoutStreaming(size: Int): Task[List[RestExampleData]] + @GET def exampleEndpointWithoutStreaming(size: RestResponseSize): Task[List[RestExampleData]] } object RestTestApi extends DefaultRestApiCompanion[RestTestApi] { final class Impl extends RestTestApi { + private var responses: Map[RestResponseSize, List[RestExampleData]] = + Map.empty - def exampleEndpoint(size: Int): Observable[RestExampleData] = - RestExampleData.generateRandomObservable(size) + def exampleEndpoint(size: RestResponseSize): Observable[RestExampleData] = + Observable.fromIterable(responses(size)) - def exampleEndpointBatch10(size: Int): Observable[RestExampleData] = - RestExampleData.generateRandomObservable(size) + def exampleEndpointBatch10(size: RestResponseSize): Observable[RestExampleData] = + Observable.fromIterable(responses(size)) - def exampleEndpointBatch500(size: Int): Observable[RestExampleData] = - RestExampleData.generateRandomObservable(size) + def exampleEndpointBatch500(size: RestResponseSize): Observable[RestExampleData] = + Observable.fromIterable(responses(size)) - def exampleEndpointWithoutStreaming(size: Int): Task[List[RestExampleData]] = - RestExampleData.generateRandomList(size) + def exampleEndpointWithoutStreaming(size: RestResponseSize): Task[List[RestExampleData]] = + Task.eval(responses(size)) + + def generateResponses(): Unit = { + this.responses = RestResponseSize.values.map(size => size -> RestExampleData.generateRandomList(size)).toMap + } } } - private def creteApiProxy(): RestTestApi = { + private def creteApiProxy(): (RestTestApi.Impl, RestTestApi) = { val apiImpl = new RestTestApi.Impl() val streamingServerHandle = RawRest.asHandleRequestWithStreaming[RestTestApi](apiImpl) val streamingClientHandler = new RawRest.RestRequestHandler { @@ -50,84 +57,88 @@ private object StreamingRestApi { override def handleRequestStream(request: RestRequest): Task[StreamedRestResponse] = streamingServerHandle(request).map(_.asInstanceOf[StreamedRestResponse]) } - RawRest.fromHandleRequestWithStreaming[RestTestApi](streamingClientHandler) + (apiImpl, RawRest.fromHandleRequestWithStreaming[RestTestApi](streamingClientHandler)) } } -@OutputTimeUnit(TimeUnit.MILLISECONDS) +@OutputTimeUnit(TimeUnit.SECONDS) @BenchmarkMode(Array(Mode.Throughput)) @State(Scope.Thread) class StreamingRestApi { implicit def scheduler: Scheduler = Scheduler.global - private final val proxy = StreamingRestApi.creteApiProxy() + private final val (impl, proxy) = StreamingRestApi.creteApiProxy() + @Setup(Level.Trial) + def setup(): Unit = { + this.impl.generateResponses() + } @Benchmark def smallArray(): Unit = { - waitStreamingEndpoint(10) + waitStreamingEndpoint(RestResponseSize.Small) } @Benchmark def mediumArray(): Unit = { - waitStreamingEndpoint(200) + waitStreamingEndpoint(RestResponseSize.Medium) } @Benchmark def hugeArray(): Unit = { - waitStreamingEndpoint(5000) + waitStreamingEndpoint(RestResponseSize.Huge) } @Benchmark def smallArrayBatch10(): Unit = { - wait(this.proxy.exampleEndpointBatch10(10).toListL) + wait(this.proxy.exampleEndpointBatch10(RestResponseSize.Small).toListL) } @Benchmark def mediumArrayBatch10(): Unit = { - wait(this.proxy.exampleEndpointBatch10(200).toListL) + wait(this.proxy.exampleEndpointBatch10(RestResponseSize.Medium).toListL) } @Benchmark def hugeArrayBatch10(): Unit = { - wait(this.proxy.exampleEndpointBatch10(5000).toListL) + wait(this.proxy.exampleEndpointBatch10(RestResponseSize.Huge).toListL) } @Benchmark def smallArrayBatch500(): Unit = { - wait(this.proxy.exampleEndpointBatch500(10).toListL) + wait(this.proxy.exampleEndpointBatch500(RestResponseSize.Small).toListL) } @Benchmark def mediumArrayBatch500(): Unit = { - wait(this.proxy.exampleEndpointBatch500(200).toListL) + wait(this.proxy.exampleEndpointBatch500(RestResponseSize.Medium).toListL) } @Benchmark def hugeArrayBatch500(): Unit = { - wait(this.proxy.exampleEndpointBatch500(5000).toListL) + wait(this.proxy.exampleEndpointBatch500(RestResponseSize.Huge).toListL) } @Benchmark def smallArrayWithoutStreaming(): Unit = { - waitEndpointWithoutStreaming(10) + waitEndpointWithoutStreaming(RestResponseSize.Small) } @Benchmark def mediumArrayWithoutStreaming(): Unit = { - waitEndpointWithoutStreaming(200) + waitEndpointWithoutStreaming(RestResponseSize.Medium) } @Benchmark def hugeArrayWithoutStreaming(): Unit = { - waitEndpointWithoutStreaming(5000) + waitEndpointWithoutStreaming(RestResponseSize.Huge) } - private def waitEndpointWithoutStreaming(samples: Int): Unit = { + private def waitEndpointWithoutStreaming(samples: RestResponseSize): Unit = { wait(this.proxy.exampleEndpointWithoutStreaming(samples)) } - private def waitStreamingEndpoint(samples: Int): Unit = { + private def waitStreamingEndpoint(samples: RestResponseSize): Unit = { wait(this.proxy.exampleEndpoint(samples).toListL) } From 265a4ef918085d3a0806b6960aab9ff5256a63b6 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 3 Jun 2025 23:22:10 +0000 Subject: [PATCH 108/162] Update sbt-ci-release to 1.11.1 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 32d66d6de..2888c5166 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -9,4 +9,4 @@ addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.4") addSbtPlugin("org.jetbrains.scala" % "sbt-ide-settings" % "1.1.2") addSbtPlugin("com.github.sbt" % "sbt-less" % "2.0.1") addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.1") -addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.9.3") +addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.11.1") From 586e67453a7408deee66bbabd571d8ed72771dba Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 3 Jun 2025 23:22:14 +0000 Subject: [PATCH 109/162] Update sbt, scripted-plugin to 1.11.1 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index d9109d9ee..ae2790d6c 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ # suppress inspection "UnusedProperty" -sbt.version=1.10.11 +sbt.version=1.11.1 From 2dd60c59b436deaf2d77bb6ef9702e2976a48845 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 3 Jun 2025 23:22:20 +0000 Subject: [PATCH 110/162] Update jetty-client to 12.0.22 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index facf3ebda..1e6482b73 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -28,7 +28,7 @@ object Dependencies { val scalaLoggingVersion = "3.9.5" - val jettyVersion = "12.0.21" + val jettyVersion = "12.0.22" val typesafeConfigVersion = "1.4.3" val flexmarkVersion = "0.64.8" val logbackVersion = "1.3.15" From c22c4e53b600a6a69d7ce9ee9d9f93b13b4b33e4 Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Wed, 4 Jun 2025 22:27:25 +0200 Subject: [PATCH 111/162] Sonatype Central is now the default --- build.sbt | 1 - 1 file changed, 1 deletion(-) diff --git a/build.sbt b/build.sbt index 9d8e58b47..f1184776c 100644 --- a/build.sbt +++ b/build.sbt @@ -11,7 +11,6 @@ Global / excludeLintKeys ++= Set(ideOutputDirectory, ideSkipProject) inThisBuild(Seq( organization := "io.udash", resolvers += Resolver.defaultLocal, - sonatypeCredentialHost := Sonatype.sonatypeCentralHost, homepage := Some(url("https://udash.io")), licenses := Seq(License.Apache2), scmInfo := Some(ScmInfo( From 2ea9234052d9f517d3d04fbc217e49e7df05412e Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Wed, 4 Jun 2025 23:13:58 +0200 Subject: [PATCH 112/162] MergeArraysBenchmark --- .../io/udash/rest/MergeArraysBenchmark.scala | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 benchmarks/src/main/scala/io/udash/rest/MergeArraysBenchmark.scala diff --git a/benchmarks/src/main/scala/io/udash/rest/MergeArraysBenchmark.scala b/benchmarks/src/main/scala/io/udash/rest/MergeArraysBenchmark.scala new file mode 100644 index 000000000..e6be6273d --- /dev/null +++ b/benchmarks/src/main/scala/io/udash/rest/MergeArraysBenchmark.scala @@ -0,0 +1,31 @@ +package io.udash.rest + +import monix.execution.Scheduler +import monix.reactive.Observable +import org.openjdk.jmh.annotations.{Benchmark, BenchmarkMode, Fork, Mode, Scope, State} +import com.avsystem.commons.concurrent.ObservableExtensions.* + +import java.io.ByteArrayOutputStream +import scala.util.Random + +@Fork(1) +@BenchmarkMode(Array(Mode.Throughput)) +@State(Scope.Benchmark) +class MergeArraysBenchmark { + + import Scheduler.Implicits.global + + private final val data: Observable[Array[Byte]] = + Observable.repeatEval(Random.nextBytes(1024)).take(32) + + @Benchmark + def mergeByteArrayOutputStream: Array[Byte] = + data.foldLeftL(new ByteArrayOutputStream()) { case (acc, elem) => + acc.write(elem) + acc + }.map(_.toByteArray).runSyncUnsafe() + + @Benchmark + def mergeByteArrayToL: Array[Byte] = + data.flatMap(Observable.fromIterable(_)).toL(Array).runSyncUnsafe() +} \ No newline at end of file From 63fdb6959038256b9f1406bdbfb82eed6b0bae5b Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Wed, 4 Jun 2025 23:14:15 +0200 Subject: [PATCH 113/162] Speedup benchmarks with just 1 fork --- benchmarks/src/main/scala/io/udash/rest/RestApi.scala | 1 + benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala | 1 + 2 files changed, 2 insertions(+) diff --git a/benchmarks/src/main/scala/io/udash/rest/RestApi.scala b/benchmarks/src/main/scala/io/udash/rest/RestApi.scala index 250687ca4..23f3dac70 100644 --- a/benchmarks/src/main/scala/io/udash/rest/RestApi.scala +++ b/benchmarks/src/main/scala/io/udash/rest/RestApi.scala @@ -40,6 +40,7 @@ private object RestApi { @OutputTimeUnit(TimeUnit.SECONDS) @BenchmarkMode(Array(Mode.Throughput)) @State(Scope.Thread) +@Fork(1) class RestApi { implicit def scheduler: Scheduler = Scheduler.global diff --git a/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala b/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala index 6ad936941..78875a281 100644 --- a/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala +++ b/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala @@ -65,6 +65,7 @@ private object StreamingRestApi { @OutputTimeUnit(TimeUnit.SECONDS) @BenchmarkMode(Array(Mode.Throughput)) @State(Scope.Thread) +@Fork(1) class StreamingRestApi { implicit def scheduler: Scheduler = Scheduler.global private final val (impl, proxy) = StreamingRestApi.creteApiProxy() From 6ce298c6f88ba6a362fc28acf335b2a2d66aa96d Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Wed, 4 Jun 2025 23:14:40 +0200 Subject: [PATCH 114/162] Use foreachL instead of .consumeWith --- rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala index 2e8e96d52..c569a3532 100644 --- a/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala +++ b/rest/.jvm/src/main/scala/io/udash/rest/RestServlet.scala @@ -150,17 +150,17 @@ class RestServlet( case binary: StreamedBody.RawBinary => response.setContentType(binary.contentType) binary.content - .consumeWith(Consumer.foreach { chunk => + .foreachL { chunk => response.getOutputStream.write(chunk) response.getOutputStream.flush() - }) + } case jsonList: StreamedBody.JsonList => response.setContentType(jsonList.contentType) jsonList.elements .bufferTumbling(jsonList.customBatchSize.getOrElse(defaultStreamingBatchSize)) .switchIfEmpty(Observable(Seq.empty)) .zipWithIndex - .consumeWith(Consumer.foreach { case (batch, idx) => + .foreachL { case (batch, idx) => val firstBatch = idx == 0 if (firstBatch) { response.getOutputStream.write("[".getBytes(jsonList.charset)) @@ -176,7 +176,7 @@ class RestServlet( response.getOutputStream.write(e.value.getBytes(jsonList.charset)) } response.getOutputStream.flush() - }) + } .map(_ => response.getOutputStream.write("]".getBytes(jsonList.charset))) } }.onErrorHandle { From bf22959b46982543fe5bd03e15c36c87e0160e7a Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Wed, 4 Jun 2025 23:14:51 +0200 Subject: [PATCH 115/162] Use larger example in chunk tests --- rest/src/test/scala/io/udash/rest/SomeServerApiImpl.scala | 2 +- rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rest/src/test/scala/io/udash/rest/SomeServerApiImpl.scala b/rest/src/test/scala/io/udash/rest/SomeServerApiImpl.scala index 488442d5c..575a606ae 100644 --- a/rest/src/test/scala/io/udash/rest/SomeServerApiImpl.scala +++ b/rest/src/test/scala/io/udash/rest/SomeServerApiImpl.scala @@ -18,7 +18,7 @@ final class SomeServerApiImpl { @GET def streamBinary(chunkSize: Int): Observable[Array[Byte]] = { - val content = "HelloWorld".getBytes + val content = ("HelloWorld" * 100).getBytes Observable.fromIterable(content.grouped(chunkSize).toSeq) } diff --git a/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala b/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala index 0c606f2c7..38fd115bb 100644 --- a/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/raw/ServerImplApiTest.scala @@ -341,7 +341,7 @@ class ServerImplApiTest extends AnyFunSuite with ScalaFutures { .map(bytes => new String(bytes)) .toListL.runToFuture - chunksFuture.futureValue.mkString("") == "HelloWorld" + chunksFuture.futureValue.mkString("") == "HelloWorld" * 100 } ) } From d4192a5e642738359dbac60218972eed63f9793f Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Wed, 4 Jun 2025 23:17:01 +0200 Subject: [PATCH 116/162] Switch to def in io.udash.rest.raw.StreamedBody.JsonList#contentType --- rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala b/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala index aefa15aa0..025071c9c 100644 --- a/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala +++ b/rest/src/main/scala/io/udash/rest/raw/StreamedBody.scala @@ -50,7 +50,7 @@ object StreamedBody extends StreamedBodyLowPrio { charset: String = HttpBody.Utf8Charset, customBatchSize: Opt[Int] = Opt.Empty, ) extends NonEmpty { - val contentType: String = s"${HttpBody.JsonType};charset=$charset" + def contentType: String = s"${HttpBody.JsonType};charset=$charset" } /** From ebe6ed3b60fe9e267ae6738d3fddd4a6f7b1a709 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 10 Jun 2025 03:53:05 +0000 Subject: [PATCH 117/162] Update sbt, scripted-plugin to 1.11.2 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index ae2790d6c..5c3bf38b3 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ # suppress inspection "UnusedProperty" -sbt.version=1.11.1 +sbt.version=1.11.2 From 1e814cf81a0afba31aa40d8519e873ada9aaabee Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Wed, 11 Jun 2025 11:07:54 +0200 Subject: [PATCH 118/162] Benchmarks for binary streaming mode --- .../main/scala/io/udash/rest/RestApi.scala | 46 ++++-- .../scala/io/udash/rest/RestExampleData.scala | 19 +-- .../io/udash/rest/StreamingRestApi.scala | 134 +++++++++++++----- 3 files changed, 144 insertions(+), 55 deletions(-) diff --git a/benchmarks/src/main/scala/io/udash/rest/RestApi.scala b/benchmarks/src/main/scala/io/udash/rest/RestApi.scala index 23f3dac70..64c484617 100644 --- a/benchmarks/src/main/scala/io/udash/rest/RestApi.scala +++ b/benchmarks/src/main/scala/io/udash/rest/RestApi.scala @@ -1,11 +1,13 @@ package io.udash.rest +import com.avsystem.commons.serialization.json.JsonStringOutput import io.udash.rest.RestExampleData.RestResponseSize import io.udash.rest.raw.RawRest import monix.eval.Task import monix.execution.Scheduler import org.openjdk.jmh.annotations.* +import java.nio.charset.StandardCharsets import java.util.concurrent.TimeUnit import scala.concurrent.Await import scala.concurrent.duration.Duration @@ -13,19 +15,24 @@ import scala.concurrent.duration.Duration private object RestApi { trait RestTestApi { @GET def exampleEndpoint(size: RestResponseSize): Task[List[RestExampleData]] + @GET def exampleBinaryEndpoint(size: RestResponseSize): Task[List[Array[Byte]]] } object RestTestApi extends DefaultRestApiCompanion[RestTestApi] { final class Impl extends RestTestApi { - private var responses: Map[RestResponseSize, List[RestExampleData]] = - Map.empty + private var responses: Map[RestResponseSize, List[RestExampleData]] = Map.empty def exampleEndpoint(size: RestResponseSize): Task[List[RestExampleData]] = - Task.eval(responses(size)) + Task.eval(getResponse(size)) - def generateResponses(): Unit = { + override def exampleBinaryEndpoint(size: RestResponseSize): Task[List[Array[Byte]]] = + Task.eval(getResponse(size).iterator.map(JsonStringOutput.write(_).getBytes(StandardCharsets.UTF_8)).toList) + + private def getResponse(size: RestResponseSize): List[RestExampleData] = + responses(size) + + def generateResponses(): Unit = this.responses = RestResponseSize.values.map(size => size -> RestExampleData.generateRandomList(size)).toMap - } } } @@ -52,21 +59,38 @@ class RestApi { } @Benchmark - def smallArray(): Unit = { + def smallArrayJsonList(): Unit = { waitEndpoint(RestResponseSize.Small) } @Benchmark - def mediumArray(): Unit = { + def mediumArrayJsonList(): Unit = { waitEndpoint(RestResponseSize.Medium) } @Benchmark - def hugeArray(): Unit = { + def hugeArrayJsonList(): Unit = { waitEndpoint(RestResponseSize.Huge) } - private def waitEndpoint(size: RestResponseSize): Unit = { - Await.result(this.proxy.exampleEndpoint(size).runToFuture, Duration.apply(10, TimeUnit.SECONDS)) + @Benchmark + def smallArrayBinary(): Unit = { + waitEndpointBinary(RestResponseSize.Small) + } + + @Benchmark + def mediumArrayBinary(): Unit = { + waitEndpointBinary(RestResponseSize.Medium) } -} \ No newline at end of file + + @Benchmark + def hugeArrayBinary(): Unit = { + waitEndpointBinary(RestResponseSize.Huge) + } + + private def waitEndpoint(size: RestResponseSize): Unit = + Await.result(this.proxy.exampleEndpoint(size).runToFuture, Duration.apply(10, TimeUnit.SECONDS)) + + private def waitEndpointBinary(size: RestResponseSize): Unit = + Await.result(this.proxy.exampleBinaryEndpoint(size).runToFuture, Duration.apply(10, TimeUnit.SECONDS)) +} diff --git a/benchmarks/src/main/scala/io/udash/rest/RestExampleData.scala b/benchmarks/src/main/scala/io/udash/rest/RestExampleData.scala index 96fa83091..cb86b4999 100644 --- a/benchmarks/src/main/scala/io/udash/rest/RestExampleData.scala +++ b/benchmarks/src/main/scala/io/udash/rest/RestExampleData.scala @@ -1,29 +1,24 @@ package io.udash.rest -import com.avsystem.commons.misc.{AbstractValueEnum, EnumCtx, ValueEnum} -import monix.eval.Task -import monix.reactive.Observable +import com.avsystem.commons.misc.{AbstractValueEnum, EnumCtx} import scala.util.Random -case class RestExampleData(number: Long, string: String) - -object RestExampleData extends RestDataCompanion[RestExampleData]{ +final case class RestExampleData(number: Long, string: String) +object RestExampleData extends RestDataCompanion[RestExampleData] { final case class RestResponseSize(value: Int)(implicit enumCtx: EnumCtx) extends AbstractValueEnum object RestResponseSize extends RestValueEnumCompanion[RestResponseSize] { final val Small: Value = new RestResponseSize(10) - final val Medium: Value = new RestResponseSize(200) - final val Huge: Value = new RestResponseSize(5000) + final val Medium: Value = new RestResponseSize(500) + final val Huge: Value = new RestResponseSize(10000) } - private def random() = { + private def random() = RestExampleData( Random.nextLong(), Iterator.continually(Random.nextPrintableChar()).take(200).mkString ) - } def generateRandomList(size: RestResponseSize): List[RestExampleData] = Range(0, size.value).toList.map(_ => RestExampleData.random()) - -} \ No newline at end of file +} diff --git a/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala b/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala index 78875a281..6b6ccb245 100644 --- a/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala +++ b/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala @@ -1,5 +1,6 @@ package io.udash.rest +import com.avsystem.commons.serialization.json.JsonStringOutput import io.udash.rest.RestExampleData.RestResponseSize import io.udash.rest.raw.{RawRest, RestRequest, RestResponse, StreamedRestResponse} import monix.eval.Task @@ -7,6 +8,7 @@ import monix.execution.Scheduler import monix.reactive.Observable import org.openjdk.jmh.annotations.* +import java.nio.charset.StandardCharsets import java.util.concurrent.TimeUnit import scala.concurrent.Await import scala.concurrent.duration.Duration @@ -14,36 +16,56 @@ import scala.concurrent.duration.Duration private object StreamingRestApi { trait RestTestApi { @GET def exampleEndpoint(size: RestResponseSize): Observable[RestExampleData] + @GET def exampleEndpointBinary(size: RestResponseSize): Observable[Array[Byte]] @streamingResponseBatchSize(10) @GET def exampleEndpointBatch10(size: RestResponseSize): Observable[RestExampleData] + @streamingResponseBatchSize(10) + @GET def exampleEndpointBatch10Binary(size: RestResponseSize): Observable[Array[Byte]] + @streamingResponseBatchSize(500) @GET def exampleEndpointBatch500(size: RestResponseSize): Observable[RestExampleData] + @streamingResponseBatchSize(500) + @GET def exampleEndpointBatch500Binary(size: RestResponseSize): Observable[Array[Byte]] + @GET def exampleEndpointWithoutStreaming(size: RestResponseSize): Task[List[RestExampleData]] } object RestTestApi extends DefaultRestApiCompanion[RestTestApi] { final class Impl extends RestTestApi { - private var responses: Map[RestResponseSize, List[RestExampleData]] = - Map.empty + private var responses: Map[RestResponseSize, List[RestExampleData]] = Map.empty def exampleEndpoint(size: RestResponseSize): Observable[RestExampleData] = - Observable.fromIterable(responses(size)) + Observable.fromIterable(getResponse(size)) + + def exampleEndpointBinary(size: RestResponseSize): Observable[Array[Byte]] = + getResponseBinary(size) def exampleEndpointBatch10(size: RestResponseSize): Observable[RestExampleData] = - Observable.fromIterable(responses(size)) + Observable.fromIterable(getResponse(size)) + + def exampleEndpointBatch10Binary(size: RestResponseSize): Observable[Array[Byte]] = + getResponseBinary(size) def exampleEndpointBatch500(size: RestResponseSize): Observable[RestExampleData] = - Observable.fromIterable(responses(size)) + Observable.fromIterable(getResponse(size)) + + def exampleEndpointBatch500Binary(size: RestResponseSize): Observable[Array[Byte]] = + getResponseBinary(size) def exampleEndpointWithoutStreaming(size: RestResponseSize): Task[List[RestExampleData]] = - Task.eval(responses(size)) + Task.eval(getResponse(size)) + + private def getResponse(size: RestResponseSize): List[RestExampleData] = + responses(size) + + private def getResponseBinary(size: RestResponseSize): Observable[Array[Byte]] = + Observable.fromIterable(getResponse(size)).map(JsonStringOutput.write(_).getBytes(StandardCharsets.UTF_8)) - def generateResponses(): Unit = { + def generateResponses(): Unit = this.responses = RestResponseSize.values.map(size => size -> RestExampleData.generateRandomList(size)).toMap - } } } @@ -76,48 +98,93 @@ class StreamingRestApi { } @Benchmark - def smallArray(): Unit = { + def smallArrayJsonList(): Unit = { waitStreamingEndpoint(RestResponseSize.Small) } @Benchmark - def mediumArray(): Unit = { + def mediumArrayJsonList(): Unit = { waitStreamingEndpoint(RestResponseSize.Medium) } @Benchmark - def hugeArray(): Unit = { + def hugeArrayJsonList(): Unit = { waitStreamingEndpoint(RestResponseSize.Huge) } @Benchmark - def smallArrayBatch10(): Unit = { - wait(this.proxy.exampleEndpointBatch10(RestResponseSize.Small).toListL) + def smallArrayBinary(): Unit = { + waitStreamingEndpointBinary(RestResponseSize.Small) + } + + @Benchmark + def mediumArrayBinary(): Unit = { + waitStreamingEndpointBinary(RestResponseSize.Medium) + } + + @Benchmark + def hugeArrayBinary(): Unit = { + waitStreamingEndpointBinary(RestResponseSize.Huge) + } + + @Benchmark + def smallArrayBatch10JsonList(): Unit = { + waitObservable(this.proxy.exampleEndpointBatch10(RestResponseSize.Small)) + } + + @Benchmark + def mediumArrayBatch10JsonList(): Unit = { + waitObservable(this.proxy.exampleEndpointBatch10(RestResponseSize.Medium)) } @Benchmark - def mediumArrayBatch10(): Unit = { - wait(this.proxy.exampleEndpointBatch10(RestResponseSize.Medium).toListL) + def hugeArrayBatch10JsonList(): Unit = { + waitObservable(this.proxy.exampleEndpointBatch10(RestResponseSize.Huge)) } @Benchmark - def hugeArrayBatch10(): Unit = { - wait(this.proxy.exampleEndpointBatch10(RestResponseSize.Huge).toListL) + def smallArrayBatch10Binary(): Unit = { + waitObservable(this.proxy.exampleEndpointBatch10Binary(RestResponseSize.Small)) } @Benchmark - def smallArrayBatch500(): Unit = { - wait(this.proxy.exampleEndpointBatch500(RestResponseSize.Small).toListL) + def mediumArrayBatch10Binary(): Unit = { + waitObservable(this.proxy.exampleEndpointBatch10Binary(RestResponseSize.Medium)) } @Benchmark - def mediumArrayBatch500(): Unit = { - wait(this.proxy.exampleEndpointBatch500(RestResponseSize.Medium).toListL) + def hugeArrayBatch10Binary(): Unit = { + waitObservable(this.proxy.exampleEndpointBatch10Binary(RestResponseSize.Huge)) } @Benchmark - def hugeArrayBatch500(): Unit = { - wait(this.proxy.exampleEndpointBatch500(RestResponseSize.Huge).toListL) + def smallArrayBatch500JsonList(): Unit = { + waitObservable(this.proxy.exampleEndpointBatch500(RestResponseSize.Small)) + } + + @Benchmark + def mediumArrayBatch500JsonList(): Unit = { + waitObservable(this.proxy.exampleEndpointBatch500(RestResponseSize.Medium)) + } + + @Benchmark + def hugeArrayBatch500JsonList(): Unit = { + waitObservable(this.proxy.exampleEndpointBatch500(RestResponseSize.Huge)) + } + + @Benchmark + def smallArrayBatch500Binary(): Unit = { + waitObservable(this.proxy.exampleEndpointBatch500Binary(RestResponseSize.Small)) + } + + @Benchmark + def mediumArrayBatch500Binary(): Unit = { + waitObservable(this.proxy.exampleEndpointBatch500Binary(RestResponseSize.Medium)) + } + + @Benchmark + def hugeArrayBatch500Binary(): Unit = { + waitObservable(this.proxy.exampleEndpointBatch500Binary(RestResponseSize.Huge)) } @Benchmark @@ -135,15 +202,18 @@ class StreamingRestApi { waitEndpointWithoutStreaming(RestResponseSize.Huge) } - private def waitEndpointWithoutStreaming(samples: RestResponseSize): Unit = { + private def waitEndpointWithoutStreaming(samples: RestResponseSize): Unit = wait(this.proxy.exampleEndpointWithoutStreaming(samples)) - } - private def waitStreamingEndpoint(samples: RestResponseSize): Unit = { - wait(this.proxy.exampleEndpoint(samples).toListL) - } + private def waitStreamingEndpoint(samples: RestResponseSize): Unit = + wait(this.proxy.exampleEndpoint(samples).completedL) - private def wait[T](task: Task[List[T]]): Unit = { - Await.result(task.runToFuture, Duration.apply(10, TimeUnit.SECONDS)) - } -} \ No newline at end of file + private def waitStreamingEndpointBinary(samples: RestResponseSize): Unit = + wait(this.proxy.exampleEndpointBinary(samples).completedL) + + private def wait[T](task: Task[T]): Unit = + Await.result(task.runToFuture, Duration.apply(15, TimeUnit.SECONDS)) + + private def waitObservable[T](obs: Observable[T]): Unit = + Await.result(obs.completedL.runToFuture, Duration.apply(15, TimeUnit.SECONDS)) +} From 541ef2d6e6d94f14a7664ac2c985b5598d6fe3b2 Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Wed, 11 Jun 2025 13:41:11 +0200 Subject: [PATCH 119/162] Add logging to client response chunk handling --- .../main/scala/io/udash/rest/jetty/JettyRestClient.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala index 8ab41999d..f1bb959c9 100644 --- a/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala +++ b/rest/jetty/src/main/scala/io/udash/rest/jetty/JettyRestClient.scala @@ -4,6 +4,7 @@ package rest.jetty import com.avsystem.commons.* import com.avsystem.commons.annotation.explicitGenerics import com.avsystem.commons.serialization.json.{JsonReader, JsonStringInput} +import com.typesafe.scalalogging.LazyLogging import io.udash.rest.raw.* import io.udash.rest.util.Utils import io.udash.utils.URLEncoder @@ -36,7 +37,7 @@ final class JettyRestClient( client: HttpClient, defaultMaxResponseLength: Int = JettyRestClient.DefaultMaxResponseLength, defaultTimeout: Duration = JettyRestClient.DefaultTimeout, -) { +) extends LazyLogging { @explicitGenerics def create[RestApi: RawRest.AsRealRpc : RestMetadata]( @@ -145,6 +146,11 @@ final class JettyRestClient( case Ack.Continue => demander.run() case Ack.Stop => () } + .onCompleteNow { + case Failure(ex) => + logger.error("Unexpected error while processing streamed response chunk", ex) + case Success(_) => + } } override def onComplete(result: Result): Unit = From 8cce31333bc659dd17f45a3bedf96dc2cb788cea Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Wed, 11 Jun 2025 17:41:09 +0200 Subject: [PATCH 120/162] Fix naming in REST benchmarks --- .../udash/rest/{RestApi.scala => RestApiBenchmark.scala} | 8 ++++---- ...amingRestApi.scala => StreamingRestApiBenchmark.scala} | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) rename benchmarks/src/main/scala/io/udash/rest/{RestApi.scala => RestApiBenchmark.scala} (93%) rename benchmarks/src/main/scala/io/udash/rest/{StreamingRestApi.scala => StreamingRestApiBenchmark.scala} (97%) diff --git a/benchmarks/src/main/scala/io/udash/rest/RestApi.scala b/benchmarks/src/main/scala/io/udash/rest/RestApiBenchmark.scala similarity index 93% rename from benchmarks/src/main/scala/io/udash/rest/RestApi.scala rename to benchmarks/src/main/scala/io/udash/rest/RestApiBenchmark.scala index 64c484617..41b282b6a 100644 --- a/benchmarks/src/main/scala/io/udash/rest/RestApi.scala +++ b/benchmarks/src/main/scala/io/udash/rest/RestApiBenchmark.scala @@ -12,7 +12,7 @@ import java.util.concurrent.TimeUnit import scala.concurrent.Await import scala.concurrent.duration.Duration -private object RestApi { +private object RestApiBenchmark { trait RestTestApi { @GET def exampleEndpoint(size: RestResponseSize): Task[List[RestExampleData]] @GET def exampleBinaryEndpoint(size: RestResponseSize): Task[List[Array[Byte]]] @@ -36,7 +36,7 @@ private object RestApi { } } - private def creteApiProxy(): (RestTestApi.Impl, RestTestApi) = { + private def createApiProxy(): (RestTestApi.Impl, RestTestApi) = { val apiImpl = new RestTestApi.Impl() val handler = RawRest.asHandleRequest[RestTestApi](apiImpl) (apiImpl, RawRest.fromHandleRequest[RestTestApi](handler)) @@ -48,10 +48,10 @@ private object RestApi { @BenchmarkMode(Array(Mode.Throughput)) @State(Scope.Thread) @Fork(1) -class RestApi { +class RestApiBenchmark { implicit def scheduler: Scheduler = Scheduler.global - private final val (impl, proxy) = RestApi.creteApiProxy() + private final val (impl, proxy) = RestApiBenchmark.createApiProxy() @Setup(Level.Trial) def setup(): Unit = { diff --git a/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala b/benchmarks/src/main/scala/io/udash/rest/StreamingRestApiBenchmark.scala similarity index 97% rename from benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala rename to benchmarks/src/main/scala/io/udash/rest/StreamingRestApiBenchmark.scala index 6b6ccb245..22d7df3b4 100644 --- a/benchmarks/src/main/scala/io/udash/rest/StreamingRestApi.scala +++ b/benchmarks/src/main/scala/io/udash/rest/StreamingRestApiBenchmark.scala @@ -13,7 +13,7 @@ import java.util.concurrent.TimeUnit import scala.concurrent.Await import scala.concurrent.duration.Duration -private object StreamingRestApi { +private object StreamingRestApiBenchmark { trait RestTestApi { @GET def exampleEndpoint(size: RestResponseSize): Observable[RestExampleData] @GET def exampleEndpointBinary(size: RestResponseSize): Observable[Array[Byte]] @@ -88,9 +88,9 @@ private object StreamingRestApi { @BenchmarkMode(Array(Mode.Throughput)) @State(Scope.Thread) @Fork(1) -class StreamingRestApi { +class StreamingRestApiBenchmark { implicit def scheduler: Scheduler = Scheduler.global - private final val (impl, proxy) = StreamingRestApi.creteApiProxy() + private final val (impl, proxy) = StreamingRestApiBenchmark.creteApiProxy() @Setup(Level.Trial) def setup(): Unit = { From 33366c67b2662201a1c23b7e37273bbb59a2bfcd Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Wed, 11 Jun 2025 17:58:32 +0200 Subject: [PATCH 121/162] Update docs to include info about partial streaming support on client side --- guide/guide/.js/src/main/assets/pages/rest.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/guide/guide/.js/src/main/assets/pages/rest.md b/guide/guide/.js/src/main/assets/pages/rest.md index fcc44de3e..ea921c3b4 100644 --- a/guide/guide/.js/src/main/assets/pages/rest.md +++ b/guide/guide/.js/src/main/assets/pages/rest.md @@ -1035,6 +1035,9 @@ client.downloadFile("report.pdf") .foreach(chunk => outputStream.write(chunk))(monixScheduler) ``` +Jetty client provided by the framework `JettyRestClient` supports streaming only for binary data. JSON list processing +still requires receiving full payload. + ### How Streaming Works When a client makes a request to a streaming endpoint: From 804b19ed0843b6141d523730d72fb77d854d9987 Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Wed, 11 Jun 2025 18:07:06 +0200 Subject: [PATCH 122/162] mockito-scala 2.0.0 --- project/Dependencies.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index c7de7124a..e5d6fe143 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -37,7 +37,7 @@ object Dependencies { val svg4everybodyVersion = "2.1.9" val scalatestVersion = "3.2.19" - val mockitoVersion = "1.17.37" + val mockitoScalaVersion = "2.0.0" val scalaJsSecureRandomVersion = "1.0.0" // Tests only val bootstrap4Version = "4.1.3" val bootstrap4DatepickerVersion = "5.39.0" @@ -116,7 +116,7 @@ object Dependencies { "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion, "org.eclipse.jetty" % "jetty-server" % jettyVersion % Test, "org.eclipse.jetty.ee8" % "jetty-ee8-servlet" % jettyVersion % Test, - "org.mockito" %% "mockito-scala-scalatest" % mockitoVersion % Test, + "org.mockito" %% "mockito-scala-scalatest" % mockitoScalaVersion % Test, )) val restSjsDeps = restCrossDeps From d7b2364c1a3085ea3f10478d9719c71e01e4ab34 Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Wed, 11 Jun 2025 18:25:15 +0200 Subject: [PATCH 123/162] Decrease servlet timeout in test --- rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala | 2 +- rest/src/test/scala/io/udash/rest/RestTestApi.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala b/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala index 86d456f9a..569569a6e 100644 --- a/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala +++ b/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala @@ -42,7 +42,7 @@ class SttpRestCallTest extends SttpClientRestTest with RestApiTestScenarios { } class ServletTimeoutTest extends SttpClientRestTest { - override def serverTimeout: FiniteDuration = 500.millis + override def serverTimeout: FiniteDuration = 300.millis "rest method timeout" in { proxy.neverGet diff --git a/rest/src/test/scala/io/udash/rest/RestTestApi.scala b/rest/src/test/scala/io/udash/rest/RestTestApi.scala index 64ad06b33..f55e6f203 100644 --- a/rest/src/test/scala/io/udash/rest/RestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/RestTestApi.scala @@ -187,7 +187,7 @@ object RestTestApi extends DefaultRestApiCompanion[RestTestApi] { counter.increment() Future.never } - def wait(millis: Int): Future[String] = FutureUtils.delayedResult(millis.millis)(s"waited $millis ms") + def wait(millis: Int): Future[String] = FutureUtils.delayedResult(millis.millis)(s"waited $millis ms").andThenNow(println(_)) def getEntity(id: RestEntityId): Future[RestEntity] = Future.successful(RestEntity(id, s"${id.value}-name")) def complexGet(p1: Int, p2: String, h1: Int, h2: String, q1: Int, q2: String, q3: Opt[Int], c1: Int, c2: String): Future[RestEntity] = Future.successful(RestEntity(RestEntityId(s"$p1-$h1-$q1-$c1"), s"$p2-$h2-$q2-${q3.getOrElse(".")}-$c2")) From f4b76e59eb2b500cd3feb031752dc92114fbc916 Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Wed, 11 Jun 2025 18:33:24 +0200 Subject: [PATCH 124/162] Fix ServletTimeoutTest --- rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala | 2 +- rest/src/test/scala/io/udash/rest/RestTestApi.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala b/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala index 569569a6e..68e7a38b6 100644 --- a/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala +++ b/rest/.jvm/src/test/scala/io/udash/rest/SttpRestCallTest.scala @@ -48,7 +48,7 @@ class ServletTimeoutTest extends SttpClientRestTest { proxy.neverGet .failed .map { exception => - assert(exception == HttpErrorException.plain(500, "server operation timed out after 500 milliseconds")) + assert(exception == HttpErrorException.plain(500, s"server operation timed out after $serverTimeout")) } } diff --git a/rest/src/test/scala/io/udash/rest/RestTestApi.scala b/rest/src/test/scala/io/udash/rest/RestTestApi.scala index f55e6f203..64ad06b33 100644 --- a/rest/src/test/scala/io/udash/rest/RestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/RestTestApi.scala @@ -187,7 +187,7 @@ object RestTestApi extends DefaultRestApiCompanion[RestTestApi] { counter.increment() Future.never } - def wait(millis: Int): Future[String] = FutureUtils.delayedResult(millis.millis)(s"waited $millis ms").andThenNow(println(_)) + def wait(millis: Int): Future[String] = FutureUtils.delayedResult(millis.millis)(s"waited $millis ms") def getEntity(id: RestEntityId): Future[RestEntity] = Future.successful(RestEntity(id, s"${id.value}-name")) def complexGet(p1: Int, p2: String, h1: Int, h2: String, q1: Int, q2: String, q3: Opt[Int], c1: Int, c2: String): Future[RestEntity] = Future.successful(RestEntity(RestEntityId(s"$p1-$h1-$q1-$c1"), s"$p2-$h2-$q2-${q3.getOrElse(".")}-$c2")) From 8e4e8935d2e4357230be059ec19bab75b65d5868 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 14 Jun 2025 04:38:44 +0000 Subject: [PATCH 125/162] Update circe-core, circe-parser to 0.14.14 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index e5d6fe143..253516d6f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -20,7 +20,7 @@ object Dependencies { val atmosphereVersion = "2.7.15" val upickleVersion = "4.2.1" // Tests only - val circeVersion = "0.14.13" // Tests only + val circeVersion = "0.14.14" // Tests only val circeDerivationVersion = "0.13.0-M5" // Tests only val monixVersion = "3.4.1" // udash-rest only From 015975b05c02a42fba5632838d4991f630965429 Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Sat, 28 Jun 2025 16:51:30 +0200 Subject: [PATCH 126/162] scala-commons 2.23.0 --- .../scala/io/udash/auth/AuthApplication.scala | 4 ++- .../main/scala/io/udash/auth/Permission.scala | 2 +- .../scala/io/udash/auth/PermissionId.scala | 2 +- .../main/scala/io/udash/auth/exceptions.scala | 4 +++ .../properties/PropertyParameters.scala | 2 +- .../bootstrap/carousel/UdashCarousel.scala | 6 ++--- .../bootstrap/dropdown/UdashDropdown.scala | 14 +++++----- .../scala/io/udash/bindings/Bindings.scala | 8 +++--- .../io/udash/bindings/modifiers/package.scala | 2 +- .../scala/io/udash/core/Definitions.scala | 2 +- .../scala/io/udash/testing/TestState.scala | 4 +-- .../manual/PropertyErrorManualTest.scala | 2 +- .../io/udash/properties/Properties.scala | 8 +++--- .../scala/io/udash/properties/BlankTest.scala | 2 +- ...sGenCodecAndModelPropertyCreatorTest.scala | 2 +- .../properties/ImmutablePropertyTest.scala | 4 +-- .../io/udash/properties/PropertyTest.scala | 4 +-- css/src/main/scala/io/udash/css/CssBase.scala | 4 +-- .../main/scala/io/udash/css/CssStyle.scala | 16 +++++++----- .../web/commons/views/ImageFactory.scala | 2 +- .../io/udash/web/guide/RoutingStatesDef.scala | 2 +- .../web/guide/components/GuideMenu.scala | 4 +-- .../views/frontend/demos/IntroFormDemo.scala | 2 +- .../rpc/demos/ExceptionsDemoComponent.scala | 4 +-- .../io/udash/web/guide/GuideExceptions.scala | 4 +-- .../udash/web/guide/demos/activity/Call.scala | 2 +- .../guide/demos/rest/RestExampleClass.scala | 4 +-- .../guide/demos/rpc/GenCodecServerRPC.scala | 2 +- project/Dependencies.scala | 2 +- .../io/udash/rest/examples/UserApi.scala | 4 +-- .../scala/io/udash/rest/openapi/OpenApi.scala | 2 +- .../scala/io/udash/rest/raw/RestRequest.scala | 2 ++ .../io/udash/rest/raw/RestResponse.scala | 8 +++--- .../io/udash/rest/util/WithHeaders.scala | 2 +- .../io/udash/rest/CirceRestApiTest.scala | 4 +-- .../scala/io/udash/rest/RestTestApi.scala | 16 ++++++------ .../scala/io/udash/rest/TestRESTRecord.scala | 2 +- .../udash/rest/openapi/RestSchemaTest.scala | 20 +++++++------- .../rest/openapi/openapiDependencies.scala | 2 +- .../scala/io/udash/rest/raw/RawRestTest.scala | 6 ++--- .../udash/rpc/internals/UsesServerRPC.scala | 6 +++-- .../scala/io/udash/rpc/DefaultClientRPC.scala | 2 +- .../DefaultExceptionCodecRegistryTest.scala | 6 ++--- rpc/src/main/scala/io/udash/rpc/rawrpc.scala | 26 +++++++++---------- .../scala/io/udash/rpc/RpcMessagesTest.scala | 6 ++--- rpc/src/test/scala/io/udash/rpc/TestRPC.scala | 6 ++--- rpc/src/test/scala/io/udash/rpc/types.scala | 8 +++--- .../udash/testing/AsyncUdashSharedTest.scala | 3 ++- 48 files changed, 134 insertions(+), 117 deletions(-) diff --git a/auth/.js/src/main/scala/io/udash/auth/AuthApplication.scala b/auth/.js/src/main/scala/io/udash/auth/AuthApplication.scala index 0ecb4bfa7..5cf9ee494 100644 --- a/auth/.js/src/main/scala/io/udash/auth/AuthApplication.scala +++ b/auth/.js/src/main/scala/io/udash/auth/AuthApplication.scala @@ -3,7 +3,9 @@ package io.udash.auth import io.udash._ object AuthApplication { - implicit class ApplicationAuthExt[HierarchyRoot >: Null <: GState[HierarchyRoot]](val application: Application[HierarchyRoot]) extends AnyVal { + implicit final class ApplicationAuthExt[HierarchyRoot >: Null <: GState[HierarchyRoot]]( + val application: Application[HierarchyRoot] + ) extends AnyVal { /** * Adds the default listener of authorization failure in routing (redirects to provided state). * diff --git a/auth/src/main/scala/io/udash/auth/Permission.scala b/auth/src/main/scala/io/udash/auth/Permission.scala index ac03395da..b768b4410 100644 --- a/auth/src/main/scala/io/udash/auth/Permission.scala +++ b/auth/src/main/scala/io/udash/auth/Permission.scala @@ -20,7 +20,7 @@ trait Permission { object Permission { /** Single permission as a combinator resolved implicitly. */ - implicit class Single(private val permission: Permission) extends AnyVal with PermissionCombinator { + implicit final class Single(private val permission: Permission) extends AnyVal with PermissionCombinator { override def check(ctx: UserCtx): Boolean = ctx.has(permission) diff --git a/auth/src/main/scala/io/udash/auth/PermissionId.scala b/auth/src/main/scala/io/udash/auth/PermissionId.scala index b26fe3328..f495389a8 100644 --- a/auth/src/main/scala/io/udash/auth/PermissionId.scala +++ b/auth/src/main/scala/io/udash/auth/PermissionId.scala @@ -3,5 +3,5 @@ package io.udash.auth import com.avsystem.commons.serialization.{HasGenCodec, transparent} @transparent -case class PermissionId(value: String) extends AnyVal +final case class PermissionId(value: String) extends AnyVal object PermissionId extends HasGenCodec[PermissionId] diff --git a/auth/src/main/scala/io/udash/auth/exceptions.scala b/auth/src/main/scala/io/udash/auth/exceptions.scala index 59b2612b2..debc983ef 100644 --- a/auth/src/main/scala/io/udash/auth/exceptions.scala +++ b/auth/src/main/scala/io/udash/auth/exceptions.scala @@ -2,8 +2,12 @@ package io.udash.auth import com.avsystem.commons.serialization.HasGenCodec +import scala.annotation.nowarn + +@nowarn("msg=Case classes should be marked as final") case class UnauthenticatedException() extends RuntimeException(s"User has to be authenticated to access this content.") object UnauthenticatedException extends HasGenCodec[UnauthenticatedException] +@nowarn("msg=Case classes should be marked as final") case class UnauthorizedException() extends RuntimeException(s"Provided user context does not have access to this content.") object UnauthorizedException extends HasGenCodec[UnauthorizedException] diff --git a/benchmarks/.js/src/main/scala/io/udash/benchmarks/properties/PropertyParameters.scala b/benchmarks/.js/src/main/scala/io/udash/benchmarks/properties/PropertyParameters.scala index 78645cf1b..d7b034143 100644 --- a/benchmarks/.js/src/main/scala/io/udash/benchmarks/properties/PropertyParameters.scala +++ b/benchmarks/.js/src/main/scala/io/udash/benchmarks/properties/PropertyParameters.scala @@ -6,7 +6,7 @@ import japgolly.scalajs.benchmark.gui._ import scalatags.JsDom.all._ object PropertyParameters { - case class Entity(i: Int, s: String, r: Entity) + final case class Entity(i: Int, s: String, r: Entity) object Entity extends HasModelPropertyCreator[Entity] private def listenProperty(p: ReadableProperty[String]) = { diff --git a/bootstrap4/.js/src/main/scala/io/udash/bootstrap/carousel/UdashCarousel.scala b/bootstrap4/.js/src/main/scala/io/udash/bootstrap/carousel/UdashCarousel.scala index 87fa13818..2d0e8f702 100644 --- a/bootstrap4/.js/src/main/scala/io/udash/bootstrap/carousel/UdashCarousel.scala +++ b/bootstrap4/.js/src/main/scala/io/udash/bootstrap/carousel/UdashCarousel.scala @@ -254,7 +254,7 @@ object UdashCarousel { * @param keyboard Should the carousel react to keyboard events. * @param active Should the animation be active. */ - case class AnimationOptions( + final case class AnimationOptions( interval: Duration = 5 seconds, pause: PauseOption = PauseOption.Hover, wrap: Boolean = true, keyboard: Boolean = true, active: Boolean = true ) { @@ -285,7 +285,7 @@ object UdashCarousel { * @param imgSrc Slide image source url. * @param caption Slide caption content. */ -case class UdashCarouselSlide(imgSrc: Url)(caption: Modifier*) { +final case class UdashCarouselSlide(imgSrc: Url)(caption: Modifier*) { import io.udash.css.CssView._ lazy val render: Node = { @@ -296,4 +296,4 @@ case class UdashCarouselSlide(imgSrc: Url)(caption: Modifier*) { ) ).render } -} \ No newline at end of file +} diff --git a/bootstrap4/.js/src/main/scala/io/udash/bootstrap/dropdown/UdashDropdown.scala b/bootstrap4/.js/src/main/scala/io/udash/bootstrap/dropdown/UdashDropdown.scala index f3246b3f1..cd38843b3 100644 --- a/bootstrap4/.js/src/main/scala/io/udash/bootstrap/dropdown/UdashDropdown.scala +++ b/bootstrap4/.js/src/main/scala/io/udash/bootstrap/dropdown/UdashDropdown.scala @@ -133,13 +133,13 @@ object UdashDropdown { /** Default dropdown elements. */ sealed trait DefaultDropdownItem extends AbstractCase object DefaultDropdownItem { - case class Text(text: String) extends DefaultDropdownItem - case class Link(title: String, url: Url) extends DefaultDropdownItem - case class Button(title: String, clickCallback: () => Any) extends DefaultDropdownItem - case class Header(title: String) extends DefaultDropdownItem - case class Disabled(item: DefaultDropdownItem) extends DefaultDropdownItem - case class Raw(element: Element) extends DefaultDropdownItem - case class Dynamic(factory: Binding.NestedInterceptor => Element) extends DefaultDropdownItem + final case class Text(text: String) extends DefaultDropdownItem + final case class Link(title: String, url: Url) extends DefaultDropdownItem + final case class Button(title: String, clickCallback: () => Any) extends DefaultDropdownItem + final case class Header(title: String) extends DefaultDropdownItem + final case class Disabled(item: DefaultDropdownItem) extends DefaultDropdownItem + final case class Raw(element: Element) extends DefaultDropdownItem + final case class Dynamic(factory: Binding.NestedInterceptor => Element) extends DefaultDropdownItem case object Divider extends DefaultDropdownItem } diff --git a/core/.js/src/main/scala/io/udash/bindings/Bindings.scala b/core/.js/src/main/scala/io/udash/bindings/Bindings.scala index 6516295ac..7a4ee5548 100644 --- a/core/.js/src/main/scala/io/udash/bindings/Bindings.scala +++ b/core/.js/src/main/scala/io/udash/bindings/Bindings.scala @@ -1,14 +1,15 @@ package io.udash.bindings -import com.avsystem.commons._ +import com.avsystem.commons.* import io.udash.bindings.Bindings.{AttrOps, AttrPairOps, HasCssName, PropertyOps} -import io.udash.bindings.modifiers._ +import io.udash.bindings.modifiers.* import io.udash.properties.seq.ReadableSeqProperty import io.udash.properties.single.ReadableProperty -import org.scalajs.dom._ +import org.scalajs.dom.* import scalatags.JsDom import scalatags.generic.{Attr, AttrPair, AttrValue, Modifier} +import scala.annotation.nowarn import scala.scalajs.js trait Bindings { @@ -28,6 +29,7 @@ trait Bindings { implicit def seqFromNode(el: Node): Seq[Node] = Seq(el) implicit def seqFromElement(el: Element): Seq[Element] = Seq(el) + @nowarn("msg=Implicit parameters") implicit def seqNodeFromOpt[T](el: Opt[T])(implicit ev: T => Modifier[Element]): Modifier[Element] = new JsDom.all.SeqNode(el.toSeq) diff --git a/core/.js/src/main/scala/io/udash/bindings/modifiers/package.scala b/core/.js/src/main/scala/io/udash/bindings/modifiers/package.scala index f7317399f..98f36da0b 100644 --- a/core/.js/src/main/scala/io/udash/bindings/modifiers/package.scala +++ b/core/.js/src/main/scala/io/udash/bindings/modifiers/package.scala @@ -3,7 +3,7 @@ package io.udash.bindings import org.scalajs.dom.Node package object modifiers { - implicit class ElementExts(private val el: Node) extends AnyVal { + implicit final class ElementExts(private val el: Node) extends AnyVal { def replaceChildren(oldChildren: Seq[Node], newChildren: Seq[Node]): Unit = { if (oldChildren == null || oldChildren.isEmpty) newChildren.foreach(el.appendChild) else { diff --git a/core/.js/src/main/scala/io/udash/core/Definitions.scala b/core/.js/src/main/scala/io/udash/core/Definitions.scala index 781734e76..84852577a 100644 --- a/core/.js/src/main/scala/io/udash/core/Definitions.scala +++ b/core/.js/src/main/scala/io/udash/core/Definitions.scala @@ -7,7 +7,7 @@ import scalatags.generic.Modifier /** * Url wrapper - just for avoiding strings. */ -case class Url(value: String) extends AnyVal +final case class Url(value: String) extends AnyVal object Url extends HasModelPropertyCreator[Url] /** diff --git a/core/.js/src/test/scala/io/udash/testing/TestState.scala b/core/.js/src/test/scala/io/udash/testing/TestState.scala index 72bfa3a74..2f6d4cf14 100644 --- a/core/.js/src/test/scala/io/udash/testing/TestState.scala +++ b/core/.js/src/test/scala/io/udash/testing/TestState.scala @@ -8,8 +8,8 @@ sealed abstract class TestState(val parentState: Option[ContainerTestState]) ext sealed abstract class ContainerTestState(parentState: Option[ContainerTestState]) extends TestState(parentState) sealed abstract class FinalTestState(parentState: Option[ContainerTestState]) extends TestState(parentState) -case class RootState(sth: Option[Int]) extends ContainerTestState(None) -case class ClassState(arg: String, arg2: Int) extends FinalTestState(Some(RootState(None))) +final case class RootState(sth: Option[Int]) extends ContainerTestState(None) +final case class ClassState(arg: String, arg2: Int) extends FinalTestState(Some(RootState(None))) case object ObjectState extends ContainerTestState(Some(RootState(None))) case object ThrowExceptionState extends ContainerTestState(Some(RootState(None))) case object NextObjectState extends FinalTestState(Some(ObjectState)) diff --git a/core/.js/src/test/scala/manual/PropertyErrorManualTest.scala b/core/.js/src/test/scala/manual/PropertyErrorManualTest.scala index 05155d63d..3fc7d9c4c 100644 --- a/core/.js/src/test/scala/manual/PropertyErrorManualTest.scala +++ b/core/.js/src/test/scala/manual/PropertyErrorManualTest.scala @@ -58,7 +58,7 @@ object PropertyErrorManualTest { trait B { def c: Seq[C] } - case class C(d: D) + final case class C(d: D) trait D { def errorField: ClassicClass } diff --git a/core/src/main/scala/io/udash/properties/Properties.scala b/core/src/main/scala/io/udash/properties/Properties.scala index 30ab67302..08aa43ae0 100644 --- a/core/src/main/scala/io/udash/properties/Properties.scala +++ b/core/src/main/scala/io/udash/properties/Properties.scala @@ -31,20 +31,20 @@ trait Properties { } object Properties extends Properties { - class Any2Property[A] private[properties](private val value: A) extends AnyVal { + final class Any2Property[A] private[properties](private val value: A) extends AnyVal { def toProperty[B >: A : PropertyCreator]: ReadableProperty[B] = PropertyCreator[B].newImmutableProperty(value) def toModelProperty[B >: A : ModelPropertyCreator]: ReadableModelProperty[B] = ModelPropertyCreator[B].newImmutableProperty(value) } - class Any2SeqProperty[A] private[properties](private val value: Seq[A]) extends AnyVal { + final class Any2SeqProperty[A] private[properties](private val value: Seq[A]) extends AnyVal { def toSeqProperty: ReadableSeqProperty[A] = new ImmutableSeqProperty[A, Seq](value) } - class PropertySeq2SeqProperty[A] private[properties](private val value: ISeq[ReadableProperty[A]]) extends AnyVal { + final class PropertySeq2SeqProperty[A] private[properties](private val value: ISeq[ReadableProperty[A]]) extends AnyVal { def combineToSeqProperty: ReadableSeqProperty[A] = new PropertySeqCombinedReadableSeqProperty[A](value) } - class BooleanPropertyOps private[properties](private val underlying: Property[Boolean]) extends AnyVal { + final class BooleanPropertyOps private[properties](private val underlying: Property[Boolean]) extends AnyVal { /** Toggles the value of the underlying boolean-backed property. * @param force If true, the value change listeners will be fired even if value didn't change. * */ diff --git a/core/src/test/scala/io/udash/properties/BlankTest.scala b/core/src/test/scala/io/udash/properties/BlankTest.scala index 47467aec7..26510d69f 100644 --- a/core/src/test/scala/io/udash/properties/BlankTest.scala +++ b/core/src/test/scala/io/udash/properties/BlankTest.scala @@ -33,7 +33,7 @@ class BlankTest extends UdashCoreTest { } object BlankTest { - case class Entity(i: Int, s: String) + final case class Entity(i: Int, s: String) object Entity extends HasModelPropertyCreator[Entity] { implicit val default: Blank[Entity] = Blank.Simple(Entity(5, "asd")) } diff --git a/core/src/test/scala/io/udash/properties/HasGenCodecAndModelPropertyCreatorTest.scala b/core/src/test/scala/io/udash/properties/HasGenCodecAndModelPropertyCreatorTest.scala index 7ed7dbf67..f49a011c9 100644 --- a/core/src/test/scala/io/udash/properties/HasGenCodecAndModelPropertyCreatorTest.scala +++ b/core/src/test/scala/io/udash/properties/HasGenCodecAndModelPropertyCreatorTest.scala @@ -24,6 +24,6 @@ class HasGenCodecAndModelPropertyCreatorTest extends UdashCoreTest { } object HasGenCodecAndModelPropertyCreatorTest { - case class Entity(i: Int, s: String, e: Option[Entity]) + final case class Entity(i: Int, s: String, e: Option[Entity]) object Entity extends HasGenCodecAndModelPropertyCreator[Entity] } diff --git a/core/src/test/scala/io/udash/properties/ImmutablePropertyTest.scala b/core/src/test/scala/io/udash/properties/ImmutablePropertyTest.scala index 3190867bb..5c002d9b4 100644 --- a/core/src/test/scala/io/udash/properties/ImmutablePropertyTest.scala +++ b/core/src/test/scala/io/udash/properties/ImmutablePropertyTest.scala @@ -150,9 +150,9 @@ class ImmutablePropertyTest extends UdashCoreTest { } object ImmutablePropertyTest { - case class Nested(s: Nested) + final case class Nested(s: Nested) object Nested extends HasModelPropertyCreator[Nested] - case class ModelEntity(s: String, i: Seq[Int], v: Vector[Int], m: ModelEntity) + final case class ModelEntity(s: String, i: Seq[Int], v: Vector[Int], m: ModelEntity) object ModelEntity extends HasModelPropertyCreator[ModelEntity] } \ No newline at end of file diff --git a/core/src/test/scala/io/udash/properties/PropertyTest.scala b/core/src/test/scala/io/udash/properties/PropertyTest.scala index 4287652a7..803776652 100644 --- a/core/src/test/scala/io/udash/properties/PropertyTest.scala +++ b/core/src/test/scala/io/udash/properties/PropertyTest.scala @@ -1436,7 +1436,7 @@ class PropertyTest extends UdashCoreTest { } private object ReqModels { - case class Simple(i: Int, s: Simple) + final case class Simple(i: Int, s: Simple) object Simple extends HasModelPropertyCreator[Simple] trait ReqT { @@ -1444,6 +1444,6 @@ private object ReqModels { } object ReqT extends HasModelPropertyCreator[ReqT] - case class SimpleSeq(i: Seq[SimpleSeq], s: SimpleSeq) + final case class SimpleSeq(i: Seq[SimpleSeq], s: SimpleSeq) object SimpleSeq extends HasModelPropertyCreator[SimpleSeq] } diff --git a/css/src/main/scala/io/udash/css/CssBase.scala b/css/src/main/scala/io/udash/css/CssBase.scala index 53126fc8b..a7582ff68 100644 --- a/css/src/main/scala/io/udash/css/CssBase.scala +++ b/css/src/main/scala/io/udash/css/CssBase.scala @@ -188,12 +188,12 @@ trait CssBase { } object CssBase { - class AnimationNameExt(private val n: Attrs.animationName.type) extends AnyVal { + final class AnimationNameExt(private val n: Attrs.animationName.type) extends AnyVal { def apply(s: CssStyle): AV = AV(n.attr, s.classNames.mkString(" ")) } - class FontFamilyExt(private val n: Attrs.fontFamily.type) extends AnyVal { + final class FontFamilyExt(private val n: Attrs.fontFamily.type) extends AnyVal { def apply(s: CssStyle): AV = AV(n.attr, s.classNames.mkString(" ")) } diff --git a/css/src/main/scala/io/udash/css/CssStyle.scala b/css/src/main/scala/io/udash/css/CssStyle.scala index 1d022fa85..0234b6881 100644 --- a/css/src/main/scala/io/udash/css/CssStyle.scala +++ b/css/src/main/scala/io/udash/css/CssStyle.scala @@ -9,15 +9,19 @@ sealed trait CssStyle { def commonPrefixClass: Option[String] = None def classNames: Seq[String] = commonPrefixClass.toList :+ className } -case class CssStyleName(className: String) extends CssStyle -case class CssPrefixedStyleName(prefixClass: String, actualClassSuffix: String) extends CssStyle { + +final case class CssStyleName(className: String) extends CssStyle + +final case class CssPrefixedStyleName(prefixClass: String, actualClassSuffix: String) extends CssStyle { val className = s"$prefixClass-$actualClassSuffix" override val commonPrefixClass: Option[String] = Some(prefixClass) } -case class CssStyleNameWithSharedCompanion(companionClass: String, commonPrefix: String, className: String) extends CssStyle { + +final case class CssStyleNameWithSharedCompanion(companionClass: String, commonPrefix: String, className: String) extends CssStyle { override val commonPrefixClass: Option[String] = Some(commonPrefix) override def classNames: Seq[String] = Seq(companionClass, className) } -case class CssStyleImpl(className: String, impl: StyleS) extends CssStyle -case class CssKeyframes(className: String, steps: Seq[(Double, StyleS)]) extends CssStyle -case class CssFontFace(className: String, font: FontFace[Option[String]]) extends CssStyle + +final case class CssStyleImpl(className: String, impl: StyleS) extends CssStyle +final case class CssKeyframes(className: String, steps: Seq[(Double, StyleS)]) extends CssStyle +final case class CssFontFace(className: String, font: FontFace[Option[String]]) extends CssStyle diff --git a/guide/commons/.js/src/main/scala/io/udash/web/commons/views/ImageFactory.scala b/guide/commons/.js/src/main/scala/io/udash/web/commons/views/ImageFactory.scala index adbdd3b28..180abb8f3 100644 --- a/guide/commons/.js/src/main/scala/io/udash/web/commons/views/ImageFactory.scala +++ b/guide/commons/.js/src/main/scala/io/udash/web/commons/views/ImageFactory.scala @@ -40,4 +40,4 @@ object SVG { } } -case class Size(width: Int, height: Int) \ No newline at end of file +final case class Size(width: Int, height: Int) diff --git a/guide/guide/.js/src/main/scala/io/udash/web/guide/RoutingStatesDef.scala b/guide/guide/.js/src/main/scala/io/udash/web/guide/RoutingStatesDef.scala index b7123b248..1fc76593c 100644 --- a/guide/guide/.js/src/main/scala/io/udash/web/guide/RoutingStatesDef.scala +++ b/guide/guide/.js/src/main/scala/io/udash/web/guide/RoutingStatesDef.scala @@ -49,7 +49,7 @@ case object FrontendState extends ContainerRoutingState(Some(ContentState)) case object FrontendIntroState extends RoutingState(Some(FrontendState)) -case class FrontendRoutingState(additionalArgument: Option[String]) extends RoutingState(Some(FrontendState)) +final case class FrontendRoutingState(additionalArgument: Option[String]) extends RoutingState(Some(FrontendState)) case object FrontendMVPState extends RoutingState(Some(FrontendState)) diff --git a/guide/guide/.js/src/main/scala/io/udash/web/guide/components/GuideMenu.scala b/guide/guide/.js/src/main/scala/io/udash/web/guide/components/GuideMenu.scala index f51b2ef35..3999f8f5f 100644 --- a/guide/guide/.js/src/main/scala/io/udash/web/guide/components/GuideMenu.scala +++ b/guide/guide/.js/src/main/scala/io/udash/web/guide/components/GuideMenu.scala @@ -18,8 +18,8 @@ sealed trait MenuEntry { def name: String } -case class MenuContainer(override val name: String, children: Seq[MenuLink]) extends MenuEntry -case class MenuLink(override val name: String, state: RoutingState, fragment: OptArg[String] = OptArg.Empty) extends MenuEntry +final case class MenuContainer(override val name: String, children: Seq[MenuLink]) extends MenuEntry +final case class MenuLink(override val name: String, state: RoutingState, fragment: OptArg[String] = OptArg.Empty) extends MenuEntry class GuideMenu(entries: Seq[MenuEntry], property: Property[String]) { diff --git a/guide/guide/.js/src/main/scala/io/udash/web/guide/views/frontend/demos/IntroFormDemo.scala b/guide/guide/.js/src/main/scala/io/udash/web/guide/views/frontend/demos/IntroFormDemo.scala index 5067fc6fe..7a1c14a22 100644 --- a/guide/guide/.js/src/main/scala/io/udash/web/guide/views/frontend/demos/IntroFormDemo.scala +++ b/guide/guide/.js/src/main/scala/io/udash/web/guide/views/frontend/demos/IntroFormDemo.scala @@ -7,7 +7,7 @@ import scalatags.JsDom.all._ object IntroFormDemo extends AutoDemo { - case class IntroFormDemoModel(minimum: Int, between: Int, maximum: Int) + final case class IntroFormDemoModel(minimum: Int, between: Int, maximum: Int) object IntroFormDemoModel extends HasModelPropertyCreator[IntroFormDemoModel] diff --git a/guide/guide/.js/src/main/scala/io/udash/web/guide/views/rpc/demos/ExceptionsDemoComponent.scala b/guide/guide/.js/src/main/scala/io/udash/web/guide/views/rpc/demos/ExceptionsDemoComponent.scala index 3db2ae1ae..6a91d2d71 100644 --- a/guide/guide/.js/src/main/scala/io/udash/web/guide/views/rpc/demos/ExceptionsDemoComponent.scala +++ b/guide/guide/.js/src/main/scala/io/udash/web/guide/views/rpc/demos/ExceptionsDemoComponent.scala @@ -16,10 +16,10 @@ import scala.concurrent.duration.DurationLong import scala.language.postfixOps import scala.util.{Failure, Success} -case class ExceptionsDemoModel( +final case class ExceptionsDemoModel( exception: String, translatableException: TranslationKey0, - unknownException: String + unknownException: String, ) object ExceptionsDemoModel extends HasModelPropertyCreator[ExceptionsDemoModel] diff --git a/guide/shared/src/main/scala/io/udash/web/guide/GuideExceptions.scala b/guide/shared/src/main/scala/io/udash/web/guide/GuideExceptions.scala index ab67c6990..f9442fce6 100644 --- a/guide/shared/src/main/scala/io/udash/web/guide/GuideExceptions.scala +++ b/guide/shared/src/main/scala/io/udash/web/guide/GuideExceptions.scala @@ -6,8 +6,8 @@ import io.udash.i18n.TranslationKey0 import io.udash.rpc.serialization.{DefaultExceptionCodecRegistry, ExceptionCodecRegistry} object GuideExceptions { - case class ExampleException(msg: String) extends Exception(msg) - case class TranslatableExampleException(trKey: TranslationKey0) extends Exception + final case class ExampleException(msg: String) extends Exception(msg) + final case class TranslatableExampleException(trKey: TranslationKey0) extends Exception val registry: ExceptionCodecRegistry = (new DefaultExceptionCodecRegistry).setup { registry => registry.register(GenCodec.materialize[ExampleException]) diff --git a/guide/shared/src/main/scala/io/udash/web/guide/demos/activity/Call.scala b/guide/shared/src/main/scala/io/udash/web/guide/demos/activity/Call.scala index 6ec5ed263..315b3533e 100644 --- a/guide/shared/src/main/scala/io/udash/web/guide/demos/activity/Call.scala +++ b/guide/shared/src/main/scala/io/udash/web/guide/demos/activity/Call.scala @@ -2,7 +2,7 @@ package io.udash.web.guide.demos.activity import com.avsystem.commons.serialization.HasGenCodec -case class Call(rpcName: String, method: String, args: Seq[String]) { +final case class Call(rpcName: String, method: String, args: Seq[String]) { override def toString: String = s"$rpcName.$method args: ${args.mkString("[", ", ", "]")}" } object Call extends HasGenCodec[Call] \ No newline at end of file diff --git a/guide/shared/src/main/scala/io/udash/web/guide/demos/rest/RestExampleClass.scala b/guide/shared/src/main/scala/io/udash/web/guide/demos/rest/RestExampleClass.scala index 81848c625..57b91817a 100644 --- a/guide/shared/src/main/scala/io/udash/web/guide/demos/rest/RestExampleClass.scala +++ b/guide/shared/src/main/scala/io/udash/web/guide/demos/rest/RestExampleClass.scala @@ -2,8 +2,8 @@ package io.udash.web.guide.demos.rest import io.udash.rest.RestDataCompanion -case class RestExampleClass(i: Int, s: String, inner: InnerClass) +final case class RestExampleClass(i: Int, s: String, inner: InnerClass) object RestExampleClass extends RestDataCompanion[RestExampleClass] -case class InnerClass(d: Double, s: String) +final case class InnerClass(d: Double, s: String) object InnerClass extends RestDataCompanion[InnerClass] \ No newline at end of file diff --git a/guide/shared/src/main/scala/io/udash/web/guide/demos/rpc/GenCodecServerRPC.scala b/guide/shared/src/main/scala/io/udash/web/guide/demos/rpc/GenCodecServerRPC.scala index c627d5c5c..0bd6878cf 100644 --- a/guide/shared/src/main/scala/io/udash/web/guide/demos/rpc/GenCodecServerRPC.scala +++ b/guide/shared/src/main/scala/io/udash/web/guide/demos/rpc/GenCodecServerRPC.scala @@ -6,7 +6,7 @@ import io.udash.rpc._ import scala.concurrent.Future object GenCodecServerRPC extends DefaultServerRpcCompanion[GenCodecServerRPC] { - case class DemoCaseClass(i: Int, s: String, intAsDouble: Double) + final case class DemoCaseClass(i: Int, s: String, intAsDouble: Double) object DemoCaseClass extends HasGenCodec[DemoCaseClass] sealed trait Fruit diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 253516d6f..fdc0271df 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -14,7 +14,7 @@ object Dependencies { val scalaCssVersion = "1.0.0" val servletVersion = "4.0.1" - val avsCommonsVersion = "2.22.0" + val avsCommonsVersion = "2.23.0" val atmosphereJSVersion = "3.1.3" val atmosphereVersion = "2.7.15" diff --git a/rest/.jvm/src/test/scala/io/udash/rest/examples/UserApi.scala b/rest/.jvm/src/test/scala/io/udash/rest/examples/UserApi.scala index d3c47601d..89181b980 100644 --- a/rest/.jvm/src/test/scala/io/udash/rest/examples/UserApi.scala +++ b/rest/.jvm/src/test/scala/io/udash/rest/examples/UserApi.scala @@ -12,12 +12,12 @@ trait UserApi { } object UserApi extends DefaultRestApiCompanion[UserApi] -case class UserId(id: Long) extends AnyVal +final case class UserId(id: Long) extends AnyVal object UserId extends RestDataWrapperCompanion[Long, UserId] @description("Representation of system user") @example(User(UserId(0), "Fred")) -case class User(id: UserId, @description("User name") name: String) +final case class User(id: UserId, @description("User name") name: String) object User extends RestDataCompanion[User] trait GroupApi diff --git a/rest/src/main/scala/io/udash/rest/openapi/OpenApi.scala b/rest/src/main/scala/io/udash/rest/openapi/OpenApi.scala index e839e5e42..87c054048 100644 --- a/rest/src/main/scala/io/udash/rest/openapi/OpenApi.scala +++ b/rest/src/main/scala/io/udash/rest/openapi/OpenApi.scala @@ -308,7 +308,7 @@ object Schema extends HasGenObjectCodec[Schema] { Schema(allOf = List(ref), nullable = true) } - implicit class RefOrOps(private val refOrSchema: RefOr[Schema]) extends AnyVal { + implicit final class RefOrOps(private val refOrSchema: RefOr[Schema]) extends AnyVal { /** * Transforms a potential schema reference into an actual [[Schema]] by wrapping the reference into * `allOf` property of the new schema, e.g. `{"$$ref": "#/components/schemas/Entity"}` becomes diff --git a/rest/src/main/scala/io/udash/rest/raw/RestRequest.scala b/rest/src/main/scala/io/udash/rest/raw/RestRequest.scala index eafaaf09e..df8922223 100644 --- a/rest/src/main/scala/io/udash/rest/raw/RestRequest.scala +++ b/rest/src/main/scala/io/udash/rest/raw/RestRequest.scala @@ -6,6 +6,7 @@ import com.avsystem.commons.meta.* import com.avsystem.commons.misc.{AbstractValueEnum, AbstractValueEnumCompanion, EnumCtx} import com.avsystem.commons.rpc.* +import scala.annotation.nowarn import scala.util.control.NoStackTrace /** @@ -49,6 +50,7 @@ object RestParameters { final val Empty = RestParameters() } +@nowarn("msg=Case classes should be marked as final") case class HttpErrorException(code: Int, payload: HttpBody = HttpBody.Empty, cause: Throwable = null) extends RuntimeException(s"HTTP ERROR $code${payload.textualContentOpt.fold("")(p => s": $p")}", cause) with NoStackTrace { def toResponse: RestResponse = RestResponse(code, IMapping.empty, payload) diff --git a/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala b/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala index 836d0dbbb..3d34d696f 100644 --- a/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala +++ b/rest/src/main/scala/io/udash/rest/raw/RestResponse.scala @@ -40,14 +40,14 @@ object RestResponse extends RestResponseLowPrio { def plain(status: Int, message: OptArg[String] = OptArg.Empty): RestResponse = RestResponse(status, IMapping.empty, HttpBody.plain(message)) - class LazyOps(private val resp: () => RestResponse) extends AnyVal { + final class LazyOps(private val resp: () => RestResponse) extends AnyVal { def recoverHttpError: RestResponse = try resp() catch { case e: HttpErrorException => e.toResponse } } implicit def lazyOps(resp: => RestResponse): LazyOps = new LazyOps(() => resp) - implicit class TaskOps(private val asyncResp: Task[RestResponse]) extends AnyVal { + implicit final class TaskOps(private val asyncResp: Task[RestResponse]) extends AnyVal { def recoverHttpError: Task[RestResponse] = asyncResp.onErrorRecover { case e: HttpErrorException => e.toResponse @@ -171,14 +171,14 @@ object StreamedRestResponse extends StreamedRestResponseLowPrio { def fromHttpError(error: HttpErrorException): StreamedRestResponse = StreamedRestResponse(error.code, IMapping.empty, StreamedBody.fromHttpBody(error.payload)) - class LazyOps(private val resp: () => StreamedRestResponse) extends AnyVal { + final class LazyOps(private val resp: () => StreamedRestResponse) extends AnyVal { def recoverHttpError: StreamedRestResponse = try resp() catch { case e: HttpErrorException => StreamedRestResponse.fromHttpError(e) } } implicit def lazyOps(resp: => StreamedRestResponse): LazyOps = new LazyOps(() => resp) - implicit class TaskOps(private val asyncResp: Task[StreamedRestResponse]) extends AnyVal { + implicit final class TaskOps(private val asyncResp: Task[StreamedRestResponse]) extends AnyVal { def recoverHttpError: Task[StreamedRestResponse] = asyncResp.onErrorRecover { case e: HttpErrorException => StreamedRestResponse.fromHttpError(e) diff --git a/rest/src/main/scala/io/udash/rest/util/WithHeaders.scala b/rest/src/main/scala/io/udash/rest/util/WithHeaders.scala index c8d8edb82..b309c2def 100644 --- a/rest/src/main/scala/io/udash/rest/util/WithHeaders.scala +++ b/rest/src/main/scala/io/udash/rest/util/WithHeaders.scala @@ -13,7 +13,7 @@ import io.udash.rest.raw.{HttpBody, IMapping, PlainValue, RestResponse} * If you want to include this information into OpenAPI definition for method that returns `WithHeaders`, * you may use [[io.udash.rest.adjustResponse adjustResponse]] on it. */ -case class WithHeaders[+T](value: T, headers: ISeq[(String, String)]) +final case class WithHeaders[+T](value: T, headers: ISeq[(String, String)]) object WithHeaders { implicit def asResponse[T](implicit wrapped: AsRaw[HttpBody, T]): AsRaw[RestResponse, WithHeaders[T]] = { case WithHeaders(value, headers) => diff --git a/rest/src/test/scala/io/udash/rest/CirceRestApiTest.scala b/rest/src/test/scala/io/udash/rest/CirceRestApiTest.scala index 31406349c..44ef2ee30 100644 --- a/rest/src/test/scala/io/udash/rest/CirceRestApiTest.scala +++ b/rest/src/test/scala/io/udash/rest/CirceRestApiTest.scala @@ -79,10 +79,10 @@ abstract class HasCirceCustomizedCodec[T]( implicit final lazy val decoder: Decoder[T] = instances((), this).decoder(nameTransform, useDefaults, discriminator) } -case class CirceAddress(city: String, zip: String) +final case class CirceAddress(city: String, zip: String) object CirceAddress extends HasCirceCustomizedCodec[CirceAddress](_.toUpperCase) -case class CircePerson(id: Long, name: String, address: Option[CirceAddress] = None) +final case class CircePerson(id: Long, name: String, address: Option[CirceAddress] = None) object CircePerson extends HasCirceCodec[CircePerson] abstract class CirceRestApiCompanion[T]( diff --git a/rest/src/test/scala/io/udash/rest/RestTestApi.scala b/rest/src/test/scala/io/udash/rest/RestTestApi.scala index 64ad06b33..2f217fd28 100644 --- a/rest/src/test/scala/io/udash/rest/RestTestApi.scala +++ b/rest/src/test/scala/io/udash/rest/RestTestApi.scala @@ -16,7 +16,7 @@ import scala.concurrent.Future import scala.concurrent.duration.* @description("Entity identifier") -case class RestEntityId(value: String) extends AnyVal +final case class RestEntityId(value: String) extends AnyVal object RestEntityId extends RestDataWrapperCompanion[String, RestEntityId] @name("RestEntityEnumCustom") @@ -42,7 +42,7 @@ object BaseEntity extends RestDataCompanion[BaseEntity] object FlatBaseEntity extends RestDataCompanion[FlatBaseEntity] @description("REST entity") -case class RestEntity( +final case class RestEntity( @description("entity id") id: RestEntityId, @whenAbsent("anonymous") name: String = whenAbsent.value, @description("recursive optional subentity") subentity: OptArg[RestEntity] = OptArg.Empty, @@ -52,11 +52,11 @@ case class RestEntity( ) extends FlatBaseEntity object RestEntity extends RestDataCompanion[RestEntity] -case class RestOtherEntity(fuu: Boolean, kek: List[String]) extends FlatBaseEntity +final case class RestOtherEntity(fuu: Boolean, kek: List[String]) extends FlatBaseEntity case object SingletonEntity extends FlatBaseEntity -case class CustomResp(value: String) +final case class CustomResp(value: String) object CustomResp { implicit val asResponse: AsRawReal[RestResponse, CustomResp] = AsRawReal.create( cr => RestResponse(200, IMapping.create("X-Value" -> PlainValue(cr.value)), HttpBody.plain("Yes")), @@ -77,10 +77,10 @@ object CustomResp { } @description("binary bytes") -case class Bytes(bytes: Array[Byte]) extends AnyVal +final case class Bytes(bytes: Array[Byte]) extends AnyVal object Bytes extends RestDataWrapperCompanion[Array[Byte], Bytes] -case class ThirdParty(thing: Int) +final case class ThirdParty(thing: Int) object ThirdPartyImplicits { implicit val thirdPartyCodec: GenCodec[ThirdParty] = GenCodec.materialize[ThirdParty] @@ -88,10 +88,10 @@ object ThirdPartyImplicits { RestStructure.materialize[ThirdParty].standaloneSchema } -case class HasThirdParty(dur: ThirdParty) +final case class HasThirdParty(dur: ThirdParty) object HasThirdParty extends RestDataCompanionWithDeps[ThirdPartyImplicits.type, HasThirdParty] -case class ErrorWrapper[T](error: T) +final case class ErrorWrapper[T](error: T) object ErrorWrapper extends HasPolyGenCodec[ErrorWrapper] trait RestTestApi { diff --git a/rest/src/test/scala/io/udash/rest/TestRESTRecord.scala b/rest/src/test/scala/io/udash/rest/TestRESTRecord.scala index 02f0a6591..994dbc465 100644 --- a/rest/src/test/scala/io/udash/rest/TestRESTRecord.scala +++ b/rest/src/test/scala/io/udash/rest/TestRESTRecord.scala @@ -1,5 +1,5 @@ package io.udash package rest -case class TestRESTRecord(id: Option[Int], s: String) +final case class TestRESTRecord(id: Option[Int], s: String) object TestRESTRecord extends RestDataCompanion[TestRESTRecord] \ No newline at end of file diff --git a/rest/src/test/scala/io/udash/rest/openapi/RestSchemaTest.scala b/rest/src/test/scala/io/udash/rest/openapi/RestSchemaTest.scala index 18e051c05..183591cfd 100644 --- a/rest/src/test/scala/io/udash/rest/openapi/RestSchemaTest.scala +++ b/rest/src/test/scala/io/udash/rest/openapi/RestSchemaTest.scala @@ -23,7 +23,7 @@ object Dependency { } @description("kejs klass") -case class KejsKlass( +final case class KejsKlass( @name("integer") @customWa(42) int: Int, @description("serious dependency") dep: Dependency, @description("optional thing") @optionalParam opty: Opt[String] = Opt("defaultThatMustBeIgnored"), @@ -32,13 +32,13 @@ case class KejsKlass( object KejsKlass extends RestDataCompanion[KejsKlass] @description("wrapped string") -@transparent case class Wrap(str: String) +@transparent final case class Wrap(str: String) object Wrap extends RestDataCompanion[Wrap] -case class PlainGenericCC[+T](thing: T) +final case class PlainGenericCC[+T](thing: T) object PlainGenericCC extends PolyRestDataCompanion[PlainGenericCC] -case class GenCC[+T >: Null](@customWa[T](null) value: T) +final case class GenCC[+T >: Null](@customWa[T](null) value: T) object GenCC extends RestDataCompanion[GenCC[String]] final class KeyEnum(implicit enumCtx: EnumCtx) extends AbstractValueEnum @@ -48,7 +48,7 @@ object KeyEnum extends AbstractValueEnumCompanion[KeyEnum] { @flatten("tpe") sealed trait HierarchyRoot[+T] -case class HierarchyCase[+T](value: T) extends HierarchyRoot[T] +final case class HierarchyCase[+T](value: T) extends HierarchyRoot[T] object HierarchyCase { implicit val stringRestSchema: RestSchema[HierarchyCase[String]] = RestStructure.materialize[HierarchyCase[String]] match { @@ -65,8 +65,8 @@ object HierarchyRoot { @flatten("case") sealed trait FullyQualifiedHierarchy object FullyQualifiedHierarchy extends RestDataCompanionWithDeps[FullyQualifiedNames.type, FullyQualifiedHierarchy] { - case class Foo(str: String) extends FullyQualifiedHierarchy - case class Bar(int: Int) extends FullyQualifiedHierarchy + final case class Foo(str: String) extends FullyQualifiedHierarchy + final case class Bar(int: Int) extends FullyQualifiedHierarchy object Bar extends RestDataCompanionWithDeps[FullyQualifiedNames.type, Bar] case object Baz extends FullyQualifiedHierarchy } @@ -76,15 +76,15 @@ sealed trait CustomSchemaNameHierarchy object CustomSchemaNameHierarchy extends RestDataCompanion[CustomSchemaNameHierarchy] { // annotation value should be used as schema name, but NOT as type discriminator value @schemaName("CustomSchemaName123") - case class CustomSchemaName(str: String) extends CustomSchemaNameHierarchy + final case class CustomSchemaName(str: String) extends CustomSchemaNameHierarchy // annotation value should be used as both schema name and type discriminator value @name("CustomName123") - case class CustomName(str: String) extends CustomSchemaNameHierarchy + final case class CustomName(str: String) extends CustomSchemaNameHierarchy // @schemaName annotation should be used as schema name, @name annotation should be used only as type discriminator value @schemaName("CustomSchemaNameBoth") @name("CustomNameBoth123") - case class CustomNameBoth(str: String) extends CustomSchemaNameHierarchy + final case class CustomNameBoth(str: String) extends CustomSchemaNameHierarchy } class RestSchemaTest extends AnyFunSuite { diff --git a/rest/src/test/scala/io/udash/rest/openapi/openapiDependencies.scala b/rest/src/test/scala/io/udash/rest/openapi/openapiDependencies.scala index c307f261a..a815d6341 100644 --- a/rest/src/test/scala/io/udash/rest/openapi/openapiDependencies.scala +++ b/rest/src/test/scala/io/udash/rest/openapi/openapiDependencies.scala @@ -49,7 +49,7 @@ class descriptionKey(key: String, @infer i18n: I18N = infer.value) extends description(i18n.t(key)) @descriptionKey("person.desc") -case class Person( +final case class Person( @descriptionKey("name.desc") name: String ) object Person extends I18NRestDataCompanion[Person] diff --git a/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala b/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala index 16e71ee79..ec0c976f2 100644 --- a/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala +++ b/rest/src/test/scala/io/udash/rest/raw/RawRestTest.scala @@ -15,15 +15,15 @@ import org.scalatest.concurrent.ScalaFutures import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -case class UserId(id: String) extends AnyVal { +final case class UserId(id: String) extends AnyVal { override def toString: String = id } object UserId extends RestDataWrapperCompanion[String, UserId] -case class User(id: UserId, name: String) +final case class User(id: UserId, name: String) object User extends RestDataCompanion[User] -case class NonBlankString(str: String) { +final case class NonBlankString(str: String) { if (str.isBlank) { throw HttpErrorException(400, HttpBody.plain("this stuff is blank")) } diff --git a/rpc/.js/src/main/scala/io/udash/rpc/internals/UsesServerRPC.scala b/rpc/.js/src/main/scala/io/udash/rpc/internals/UsesServerRPC.scala index 9a5c86417..8b1758a29 100644 --- a/rpc/.js/src/main/scala/io/udash/rpc/internals/UsesServerRPC.scala +++ b/rpc/.js/src/main/scala/io/udash/rpc/internals/UsesServerRPC.scala @@ -1,10 +1,11 @@ package io.udash.rpc.internals -import com.avsystem.commons.SharedExtensions._ -import io.udash.rpc._ +import com.avsystem.commons.SharedExtensions.* +import io.udash.rpc.* import io.udash.utils.{CallbacksHandler, Registration} import org.scalajs.dom +import scala.annotation.nowarn import scala.concurrent.duration.{Duration, DurationInt} import scala.concurrent.{Future, Promise} import scala.scalajs.js @@ -96,5 +97,6 @@ private[rpc] trait UsesServerRPC[ServerRPCType] extends UsesRemoteRPC[ServerRPCT } object UsesServerRPC { + @nowarn("msg=Case classes should be marked as final") case class CallTimeout(callTimeout: Duration) extends RuntimeException(s"Response missing after $callTimeout.") } diff --git a/rpc/.jvm/src/main/scala/io/udash/rpc/DefaultClientRPC.scala b/rpc/.jvm/src/main/scala/io/udash/rpc/DefaultClientRPC.scala index a35141405..086bb675b 100644 --- a/rpc/.jvm/src/main/scala/io/udash/rpc/DefaultClientRPC.scala +++ b/rpc/.jvm/src/main/scala/io/udash/rpc/DefaultClientRPC.scala @@ -13,7 +13,7 @@ import scala.concurrent.ExecutionContext */ sealed trait ClientRPCTarget case object AllClients extends ClientRPCTarget -case class ClientId(id: String) extends ClientRPCTarget +final case class ClientId(id: String) extends ClientRPCTarget abstract class ClientRPC[ClientRPCType](target: ClientRPCTarget) (implicit ec: ExecutionContext) extends UsesClientRPC[ClientRPCType] { diff --git a/rpc/.jvm/src/test/scala/io/udash/rpc/DefaultExceptionCodecRegistryTest.scala b/rpc/.jvm/src/test/scala/io/udash/rpc/DefaultExceptionCodecRegistryTest.scala index f7611d372..63ef3f9fb 100644 --- a/rpc/.jvm/src/test/scala/io/udash/rpc/DefaultExceptionCodecRegistryTest.scala +++ b/rpc/.jvm/src/test/scala/io/udash/rpc/DefaultExceptionCodecRegistryTest.scala @@ -6,11 +6,11 @@ import io.udash.testing.UdashSharedTest private sealed trait RootTrait extends Throwable private sealed trait SubTrait extends RootTrait -private case class SubTraitImpl() extends SubTrait +private final case class SubTraitImpl() extends SubTrait private sealed trait SealedHierarchy extends Throwable -private case class SealedHierarchyA(a: Int) extends SealedHierarchy -private case class SealedHierarchyB(b: Double) extends SealedHierarchy +private final case class SealedHierarchyA(a: Int) extends SealedHierarchy +private final case class SealedHierarchyB(b: Double) extends SealedHierarchy class DefaultExceptionCodecRegistryTest extends UdashSharedTest with Utils { val exceptionsRegistry: ExceptionCodecRegistry = new DefaultExceptionCodecRegistry diff --git a/rpc/src/main/scala/io/udash/rpc/rawrpc.scala b/rpc/src/main/scala/io/udash/rpc/rawrpc.scala index 61c18ca52..bc8a739f2 100644 --- a/rpc/src/main/scala/io/udash/rpc/rawrpc.scala +++ b/rpc/src/main/scala/io/udash/rpc/rawrpc.scala @@ -12,11 +12,11 @@ import io.udash.rpc.utils.Logged import scala.annotation.implicitNotFound import scala.concurrent.Future -case class JsonStr(json: String) extends AnyVal +final case class JsonStr(json: String) extends AnyVal object JsonStr { implicit val codec: GenCodec[JsonStr] = GenCodec.create( i => JsonStr(i.readCustom(RawJson).getOrElse(i.readSimple().readString())), - (o, v) => if (!o.writeCustom(RawJson, v.json)) o.writeSimple().writeString(v.json) + (o, v) => if (!o.writeCustom(RawJson, v.json)) o.writeSimple().writeString(v.json), ) implicit def futureAsReal[T](implicit asReal: AsReal[JsonStr, T]): AsReal[Future[JsonStr], Future[T]] = @@ -36,10 +36,10 @@ object JsonStr { ): ImplicitNotFound[AsRaw[Future[JsonStr], Future[T]]] = ImplicitNotFound() } -case class RpcInvocation(@methodName rpcName: String, @multi args: List[JsonStr]) +final case class RpcInvocation(@methodName rpcName: String, @multi args: List[JsonStr]) object RpcInvocation extends HasGenCodec[RpcInvocation] -case class RpcFailure(remoteCause: String, remoteMessage: String) +final case class RpcFailure(remoteCause: String, remoteMessage: String) extends Exception(s"$remoteCause: $remoteMessage") object RpcFailure extends HasGenCodec[RpcFailure] @@ -56,12 +56,12 @@ sealed trait RpcRequest { object RpcRequest extends HasGenCodec[RpcRequest] /** [[RpcRequest]] which returns some value. */ -case class RpcCall(invocation: RpcInvocation, gettersChain: List[RpcInvocation], callId: String) +final case class RpcCall(invocation: RpcInvocation, gettersChain: List[RpcInvocation], callId: String) extends RpcRequest object RpcCall extends HasGenCodec[RpcCall] /** [[RpcRequest]] which returns Unit. */ -case class RpcFire(invocation: RpcInvocation, gettersChain: List[RpcInvocation]) +final case class RpcFire(invocation: RpcInvocation, gettersChain: List[RpcInvocation]) extends RpcRequest with RpcServerMessage object RpcFire extends HasGenCodec[RpcFire] @@ -74,11 +74,11 @@ object RpcResponse { } /** Message containing response for [[RpcCall]]. */ -case class RpcResponseSuccess(response: JsonStr, callId: String) extends RpcResponse +final case class RpcResponseSuccess(response: JsonStr, callId: String) extends RpcResponse /** Message reporting failure of [[RpcCall]]. */ -case class RpcResponseFailure(cause: String, errorMsg: String, callId: String) extends RpcResponse +final case class RpcResponseFailure(cause: String, errorMsg: String, callId: String) extends RpcResponse /** Message reporting exception from [[RpcCall]]. */ -case class RpcResponseException(name: String, exception: Throwable, callId: String) extends RpcResponse +final case class RpcResponseException(name: String, exception: Throwable, callId: String) extends RpcResponse object RpcResponseException { implicit def codec(implicit ecr: ExceptionCodecRegistry): GenCodec[RpcResponseException] = GenCodec.nullableObject( @@ -139,20 +139,20 @@ object ServerRawRpc extends RawRpcCompanion[ServerRawRpc] { } @allowIncomplete -case class ServerRpcMetadata[T]( +final case class ServerRpcMetadata[T]( @reifyName name: String, @multi @rpcMethodMetadata getters: Map[String, GetterMethod[_]], - @multi @rpcMethodMetadata methods: Map[String, RpcMethod[_]] + @multi @rpcMethodMetadata methods: Map[String, RpcMethod[_]], ) object ServerRpcMetadata extends RpcMetadataCompanion[ServerRpcMetadata] @allowIncomplete -case class GetterMethod[T]( +final case class GetterMethod[T]( @infer @checked resultMetadata: ServerRpcMetadata.Lazy[T] ) extends TypedMetadata[T] @allowIncomplete -case class RpcMethod[T]( +final case class RpcMethod[T]( @reifyName name: String, @isAnnotated[Logged] logged: Boolean, ) extends TypedMetadata[T] diff --git a/rpc/src/test/scala/io/udash/rpc/RpcMessagesTest.scala b/rpc/src/test/scala/io/udash/rpc/RpcMessagesTest.scala index 616c006b5..5b317c5e1 100644 --- a/rpc/src/test/scala/io/udash/rpc/RpcMessagesTest.scala +++ b/rpc/src/test/scala/io/udash/rpc/RpcMessagesTest.scala @@ -7,11 +7,11 @@ import io.udash.testing.UdashSharedTest import scala.util.Random -private case class CustomException(error: String, counter: Int) extends Throwable +private final case class CustomException(error: String, counter: Int) extends Throwable private sealed trait SealedExceptions extends Throwable -private case class SealedExceptionsA(a: Int) extends SealedExceptions -private case class SealedExceptionsB(b: Double) extends SealedExceptions +private final case class SealedExceptionsA(a: Int) extends SealedExceptions +private final case class SealedExceptionsB(b: Double) extends SealedExceptions trait RpcMessagesTestScenarios extends UdashSharedTest with Utils { val exceptionsRegistry: ExceptionCodecRegistry = new DefaultExceptionCodecRegistry diff --git a/rpc/src/test/scala/io/udash/rpc/TestRPC.scala b/rpc/src/test/scala/io/udash/rpc/TestRPC.scala index 1c0c1e0f2..485977afb 100644 --- a/rpc/src/test/scala/io/udash/rpc/TestRPC.scala +++ b/rpc/src/test/scala/io/udash/rpc/TestRPC.scala @@ -7,10 +7,10 @@ import io.udash.rpc.utils.Logged import scala.annotation.nowarn import scala.concurrent.Future -case class Record(i: Int, fuu: String) +final case class Record(i: Int, fuu: String) object Record extends HasGenCodec[Record] -case class CustomRPCException(i: Int) extends Throwable +final case class CustomRPCException(i: Int) extends Throwable object CustomRPCException extends HasGenCodec[CustomRPCException] trait RPCMethods { @@ -47,7 +47,7 @@ trait InnerClientRPC { } object InnerClientRPC extends DefaultClientRpcCompanion[InnerClientRPC] -case class External(x: Int) +final case class External(x: Int) object ExternalTypeCodec { implicit val codec: GenCodec[External] = GenCodec.transformed[External, Int](_.x, External.apply) diff --git a/rpc/src/test/scala/io/udash/rpc/types.scala b/rpc/src/test/scala/io/udash/rpc/types.scala index 7ab001970..ff74f5c64 100644 --- a/rpc/src/test/scala/io/udash/rpc/types.scala +++ b/rpc/src/test/scala/io/udash/rpc/types.scala @@ -2,16 +2,16 @@ package io.udash.rpc import com.avsystem.commons.serialization.HasGenCodec -case class TestCC(i: Int, l: Long, intAsDouble: Double, b: Boolean, s: String, list: List[Char]) +final case class TestCC(i: Int, l: Long, intAsDouble: Double, b: Boolean, s: String, list: List[Char]) object TestCC extends HasGenCodec[TestCC] -case class NestedTestCC(i: Int, t: TestCC, t2: TestCC) +final case class NestedTestCC(i: Int, t: TestCC, t2: TestCC) object NestedTestCC extends HasGenCodec[NestedTestCC] -case class DeepNestedTestCC(n: NestedTestCC, l: DeepNestedTestCC) +final case class DeepNestedTestCC(n: NestedTestCC, l: DeepNestedTestCC) object DeepNestedTestCC extends HasGenCodec[DeepNestedTestCC] -case class CompleteItem(unit: Unit, string: String, specialString: String, char: Char, boolean: Boolean, byte: Byte, short: Short, int: Int, +final case class CompleteItem(unit: Unit, string: String, specialString: String, char: Char, boolean: Boolean, byte: Byte, short: Short, int: Int, long: Long, float: Float, double: Double, binary: Array[Byte], list: List[String], set: Set[String], obj: TestCC, map: Map[String, Int]) object CompleteItem extends HasGenCodec[CompleteItem] diff --git a/utils/.js/src/test/scala/io/udash/testing/AsyncUdashSharedTest.scala b/utils/.js/src/test/scala/io/udash/testing/AsyncUdashSharedTest.scala index 334f6a796..db0014be4 100644 --- a/utils/.js/src/test/scala/io/udash/testing/AsyncUdashSharedTest.scala +++ b/utils/.js/src/test/scala/io/udash/testing/AsyncUdashSharedTest.scala @@ -7,6 +7,7 @@ import org.scalatest.{Assertion, Succeeded} import scala.concurrent.{ExecutionContext, Future, Promise} import scala.scalajs.concurrent.JSExecutionContext import scala.scalajs.js.Date +import scala.util.control.NonFatal import scala.util.{Failure, Success} trait AsyncUdashSharedTest extends AsyncUdashSharedTestBase { @@ -23,7 +24,7 @@ trait AsyncUdashSharedTest extends AsyncUdashSharedTestBase { code p.complete(Success(Succeeded)) } catch { - case ex: Throwable => + case NonFatal(ex) => lastEx = Some(ex) startTest() } From 14832d06f808a656ceb0fcd021908683d7137f79 Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Tue, 1 Jul 2025 12:24:28 +0200 Subject: [PATCH 127/162] Apply review changes: add final to selected exception classes --- .../main/scala/io/udash/auth/AuthApplication.scala | 2 +- auth/src/main/scala/io/udash/auth/exceptions.scala | 12 ++---------- .../main/scala/io/udash/rest/raw/RestRequest.scala | 8 +++----- .../scala/io/udash/rpc/internals/UsesServerRPC.scala | 4 +--- 4 files changed, 7 insertions(+), 19 deletions(-) diff --git a/auth/.js/src/main/scala/io/udash/auth/AuthApplication.scala b/auth/.js/src/main/scala/io/udash/auth/AuthApplication.scala index 5cf9ee494..2e68ad25a 100644 --- a/auth/.js/src/main/scala/io/udash/auth/AuthApplication.scala +++ b/auth/.js/src/main/scala/io/udash/auth/AuthApplication.scala @@ -4,7 +4,7 @@ import io.udash._ object AuthApplication { implicit final class ApplicationAuthExt[HierarchyRoot >: Null <: GState[HierarchyRoot]]( - val application: Application[HierarchyRoot] + private val application: Application[HierarchyRoot] ) extends AnyVal { /** * Adds the default listener of authorization failure in routing (redirects to provided state). diff --git a/auth/src/main/scala/io/udash/auth/exceptions.scala b/auth/src/main/scala/io/udash/auth/exceptions.scala index debc983ef..6a3554c4c 100644 --- a/auth/src/main/scala/io/udash/auth/exceptions.scala +++ b/auth/src/main/scala/io/udash/auth/exceptions.scala @@ -1,13 +1,5 @@ package io.udash.auth -import com.avsystem.commons.serialization.HasGenCodec +class UnauthenticatedException extends RuntimeException(s"User has to be authenticated to access this content.") -import scala.annotation.nowarn - -@nowarn("msg=Case classes should be marked as final") -case class UnauthenticatedException() extends RuntimeException(s"User has to be authenticated to access this content.") -object UnauthenticatedException extends HasGenCodec[UnauthenticatedException] - -@nowarn("msg=Case classes should be marked as final") -case class UnauthorizedException() extends RuntimeException(s"Provided user context does not have access to this content.") -object UnauthorizedException extends HasGenCodec[UnauthorizedException] +class UnauthorizedException extends RuntimeException(s"Provided user context does not have access to this content.") diff --git a/rest/src/main/scala/io/udash/rest/raw/RestRequest.scala b/rest/src/main/scala/io/udash/rest/raw/RestRequest.scala index df8922223..e2994af40 100644 --- a/rest/src/main/scala/io/udash/rest/raw/RestRequest.scala +++ b/rest/src/main/scala/io/udash/rest/raw/RestRequest.scala @@ -6,7 +6,6 @@ import com.avsystem.commons.meta.* import com.avsystem.commons.misc.{AbstractValueEnum, AbstractValueEnumCompanion, EnumCtx} import com.avsystem.commons.rpc.* -import scala.annotation.nowarn import scala.util.control.NoStackTrace /** @@ -24,14 +23,14 @@ final case class RestParameters( @multi @tagged[Path] path: List[PlainValue] = Nil, @multi @tagged[Header] @allowOptional headers: IMapping[PlainValue] = IMapping.empty, @multi @tagged[Query] @allowOptional query: Mapping[PlainValue] = Mapping.empty, - @multi @tagged[Cookie] @allowOptional cookies: Mapping[PlainValue] = Mapping.empty + @multi @tagged[Cookie] @allowOptional cookies: Mapping[PlainValue] = Mapping.empty, ) { def append(method: RestMethodMetadata[_], otherParameters: RestParameters): RestParameters = RestParameters( path ::: method.applyPathParams(otherParameters.path), headers ++ otherParameters.headers, query ++ otherParameters.query, - cookies ++ otherParameters.cookies + cookies ++ otherParameters.cookies, ) def path(values: String*): RestParameters = @@ -50,8 +49,7 @@ object RestParameters { final val Empty = RestParameters() } -@nowarn("msg=Case classes should be marked as final") -case class HttpErrorException(code: Int, payload: HttpBody = HttpBody.Empty, cause: Throwable = null) +final case class HttpErrorException(code: Int, payload: HttpBody = HttpBody.Empty, cause: Throwable = null) extends RuntimeException(s"HTTP ERROR $code${payload.textualContentOpt.fold("")(p => s": $p")}", cause) with NoStackTrace { def toResponse: RestResponse = RestResponse(code, IMapping.empty, payload) } diff --git a/rpc/.js/src/main/scala/io/udash/rpc/internals/UsesServerRPC.scala b/rpc/.js/src/main/scala/io/udash/rpc/internals/UsesServerRPC.scala index 8b1758a29..0b70f99e9 100644 --- a/rpc/.js/src/main/scala/io/udash/rpc/internals/UsesServerRPC.scala +++ b/rpc/.js/src/main/scala/io/udash/rpc/internals/UsesServerRPC.scala @@ -5,7 +5,6 @@ import io.udash.rpc.* import io.udash.utils.{CallbacksHandler, Registration} import org.scalajs.dom -import scala.annotation.nowarn import scala.concurrent.duration.{Duration, DurationInt} import scala.concurrent.{Future, Promise} import scala.scalajs.js @@ -97,6 +96,5 @@ private[rpc] trait UsesServerRPC[ServerRPCType] extends UsesRemoteRPC[ServerRPCT } object UsesServerRPC { - @nowarn("msg=Case classes should be marked as final") - case class CallTimeout(callTimeout: Duration) extends RuntimeException(s"Response missing after $callTimeout.") + final case class CallTimeout(callTimeout: Duration) extends RuntimeException(s"Response missing after $callTimeout.") } From 180063842228c716143d63b5a8666ea3d8f96140 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 2 Jul 2025 05:39:54 +0000 Subject: [PATCH 128/162] Update selenium-java to 4.34.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index fdc0271df..96a480c21 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -43,7 +43,7 @@ object Dependencies { val bootstrap4DatepickerVersion = "5.39.0" val momentJsVersion = "2.30.1" - val seleniumVersion = "4.33.0" + val seleniumVersion = "4.34.0" val webDriverManagerVersion = "6.1.0" val scalaJsBenchmarkVersion = "0.10.0" From 4af85cf23856af58b5e2ce97ac7dfcc8c33e2774 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Fri, 4 Jul 2025 03:39:19 +0000 Subject: [PATCH 129/162] Update jetty-client to 12.0.23 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 96a480c21..de64e79be 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -28,7 +28,7 @@ object Dependencies { val scalaLoggingVersion = "3.9.5" - val jettyVersion = "12.0.22" + val jettyVersion = "12.0.23" val typesafeConfigVersion = "1.4.3" val flexmarkVersion = "0.64.8" val logbackVersion = "1.3.15" From 2a57597391bd256246dfe1e38a01307aa47f7a8c Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sun, 6 Jul 2025 21:55:21 +0000 Subject: [PATCH 130/162] Update sbt, scripted-plugin to 1.11.3 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index 5c3bf38b3..b121e24fc 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ # suppress inspection "UnusedProperty" -sbt.version=1.11.2 +sbt.version=1.11.3 From 0acde1ffc93bf98b74f92502aca8293eefd0aa4a Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Wed, 9 Jul 2025 12:16:22 +0200 Subject: [PATCH 131/162] Rest wrapper implicits --- .../io/udash/rest/RestDataCompanion.scala | 51 +++++++++++-------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/rest/src/main/scala/io/udash/rest/RestDataCompanion.scala b/rest/src/main/scala/io/udash/rest/RestDataCompanion.scala index 291d52204..ae2fe4b64 100644 --- a/rest/src/main/scala/io/udash/rest/RestDataCompanion.scala +++ b/rest/src/main/scala/io/udash/rest/RestDataCompanion.scala @@ -4,7 +4,7 @@ package rest import com.avsystem.commons.meta.MacroInstances import com.avsystem.commons.misc.{AbstractValueEnumCompanion, ValueEnum, ValueOf} import com.avsystem.commons.rpc.{AsRaw, AsReal} -import com.avsystem.commons.serialization.{GenCodec, TransparentWrapperCompanion} +import com.avsystem.commons.serialization.{GenCodec, TransparentWrapperCompanion, TransparentWrapping} import io.udash.rest.openapi.* import io.udash.rest.openapi.RestStructure.NameAndAdjusters import io.udash.rest.raw.{HttpBody, JsonValue, PlainValue, RestResponse, StreamedBody, StreamedRestResponse} @@ -47,28 +47,18 @@ abstract class RestDataCompanionWithDeps[D, T](implicit ) extends AbstractRestDataCompanion[(DefaultRestImplicits, D), T]((DefaultRestImplicits, deps.value)) /** - * Base class for companion objects of wrappers over other data types (i.e. case classes with single field). - * This companion ensures instances of all the REST typeclasses (serialization, schema, etc.) for wrapping type - * assuming that these instances are available for the wrapped type. - * - * Using this base companion class makes the wrapper class effectively "transparent", i.e. as if it was annotated with - * [[com.avsystem.commons.serialization.transparent transparent]] annotation. + * These implicits must be specialized for every raw type (PlainValue, JsonValue, etc.) because + * it lifts their priority. Unfortunately, controlling implicit priority is not pretty. + * Also, it's probably good that we explicitly enable derivation only for REST-related raw types + * and not for all raw types - this avoids possible interference with other features using RPC. * - * @example - * {{{ - * case class UserId(id: String) extends AnyVal - * object UserId extends RestDataWrapperCompanion[String, UserId] - * }}} + * Seperated from [[RestDataWrapperCompanion]] to allow creating custom companion wrappers. */ -abstract class RestDataWrapperCompanion[Wrapped, T](implicit - instances: MacroInstances[DefaultRestImplicits, () => NameAndAdjusters[T]] -) extends TransparentWrapperCompanion[Wrapped, T] { - private def nameAndAdjusters: NameAndAdjusters[T] = instances(DefaultRestImplicits, this).apply() +trait RestDataWrapperImplicits[Wrapped, T] { + protected def nameAndAdjusters: NameAndAdjusters[T] + protected def wrapping: TransparentWrapping[Wrapped, T] - // These implicits must be specialized for every raw type (PlainValue, JsonValue, etc.) because - // it lifts their priority. Unfortunately, controlling implicit priority is not pretty. - // Also, it's probably good that we explicitly enable derivation only for REST-related raw types - // and not for all raw types - this avoids possible interference with other features using RPC. + private implicit def wrappingAsImplicit: TransparentWrapping[Wrapped, T] = wrapping implicit def plainAsRaw(implicit wrappedAsRaw: AsRaw[PlainValue, Wrapped]): AsRaw[PlainValue, T] = AsRaw.fromTransparentWrapping @@ -122,6 +112,27 @@ abstract class RestDataWrapperCompanion[Wrapped, T](implicit wrappedResponses.responses(resolver, ws => schemaTransform(nameAndAdjusters.restSchema(ws))) } +/** + * Base class for companion objects of wrappers over other data types (i.e. case classes with single field). + * This companion ensures instances of all the REST typeclasses (serialization, schema, etc.) for wrapping type + * assuming that these instances are available for the wrapped type. + * + * Using this base companion class makes the wrapper class effectively "transparent", i.e. as if it was annotated with + * [[com.avsystem.commons.serialization.transparent transparent]] annotation. + * + * @example + * {{{ + * case class UserId(id: String) extends AnyVal + * object UserId extends RestDataWrapperCompanion[String, UserId] + * }}} + */ +abstract class RestDataWrapperCompanion[Wrapped, T](implicit + instances: MacroInstances[DefaultRestImplicits, () => NameAndAdjusters[T]] +) extends TransparentWrapperCompanion[Wrapped, T] with RestDataWrapperImplicits[Wrapped, T] { + override protected def nameAndAdjusters: NameAndAdjusters[T] = instances(DefaultRestImplicits, this).apply() + override protected def wrapping: TransparentWrapping[Wrapped, T] = this +} + /** * Base class for companion objects of enum types [[ValueEnum]] which are used as * parameter or result types in REST API traits. Automatically provides instance of From 2a7b8c5f277f76f1d8b7cad45aaee91320683be6 Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Wed, 9 Jul 2025 12:17:22 +0200 Subject: [PATCH 132/162] Rest wrapper implicits - nits --- rest/src/main/scala/io/udash/rest/RestDataCompanion.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rest/src/main/scala/io/udash/rest/RestDataCompanion.scala b/rest/src/main/scala/io/udash/rest/RestDataCompanion.scala index ae2fe4b64..2ad146353 100644 --- a/rest/src/main/scala/io/udash/rest/RestDataCompanion.scala +++ b/rest/src/main/scala/io/udash/rest/RestDataCompanion.scala @@ -43,7 +43,8 @@ abstract class RestDataCompanion[T](implicit * It must be a singleton object type, i.e. `SomeObject.type`. */ abstract class RestDataCompanionWithDeps[D, T](implicit - deps: ValueOf[D], instances: MacroInstances[(DefaultRestImplicits, D), CodecWithStructure[T]] + deps: ValueOf[D], + instances: MacroInstances[(DefaultRestImplicits, D), CodecWithStructure[T]], ) extends AbstractRestDataCompanion[(DefaultRestImplicits, D), T]((DefaultRestImplicits, deps.value)) /** From e05803579698f176869cb2c5977fd396bf93c8ec Mon Sep 17 00:00:00 2001 From: Sebastian Haracz Date: Wed, 9 Jul 2025 12:18:29 +0200 Subject: [PATCH 133/162] Rest wrapper implicits - nits --- rest/src/main/scala/io/udash/rest/RestDataCompanion.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest/src/main/scala/io/udash/rest/RestDataCompanion.scala b/rest/src/main/scala/io/udash/rest/RestDataCompanion.scala index 2ad146353..af1f2331d 100644 --- a/rest/src/main/scala/io/udash/rest/RestDataCompanion.scala +++ b/rest/src/main/scala/io/udash/rest/RestDataCompanion.scala @@ -130,8 +130,8 @@ trait RestDataWrapperImplicits[Wrapped, T] { abstract class RestDataWrapperCompanion[Wrapped, T](implicit instances: MacroInstances[DefaultRestImplicits, () => NameAndAdjusters[T]] ) extends TransparentWrapperCompanion[Wrapped, T] with RestDataWrapperImplicits[Wrapped, T] { - override protected def nameAndAdjusters: NameAndAdjusters[T] = instances(DefaultRestImplicits, this).apply() - override protected def wrapping: TransparentWrapping[Wrapped, T] = this + override protected final def nameAndAdjusters: NameAndAdjusters[T] = instances(DefaultRestImplicits, this).apply() + override protected final def wrapping: TransparentWrapping[Wrapped, T] = this } /** From 0f552e65f24080bd3e39beb2781f3ebb00ee84e4 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Fri, 11 Jul 2025 18:28:55 +0000 Subject: [PATCH 134/162] Update commons-analyzer, commons-core, ... to 2.23.1 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index de64e79be..4e30b3ae8 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -14,7 +14,7 @@ object Dependencies { val scalaCssVersion = "1.0.0" val servletVersion = "4.0.1" - val avsCommonsVersion = "2.23.0" + val avsCommonsVersion = "2.23.1" val atmosphereJSVersion = "3.1.3" val atmosphereVersion = "2.7.15" From 7ecd277a7823a7b9598790b71ea1eaac3751fd00 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Fri, 11 Jul 2025 18:28:57 +0000 Subject: [PATCH 135/162] Update typesafe:config to 1.4.4 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index de64e79be..ed93ad987 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -29,7 +29,7 @@ object Dependencies { val scalaLoggingVersion = "3.9.5" val jettyVersion = "12.0.23" - val typesafeConfigVersion = "1.4.3" + val typesafeConfigVersion = "1.4.4" val flexmarkVersion = "0.64.8" val logbackVersion = "1.3.15" val janinoVersion = "3.1.12" From 1ed485b545a2081ab7643de28bbf14145f97060a Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Fri, 11 Jul 2025 18:28:59 +0000 Subject: [PATCH 136/162] Update webdrivermanager to 6.1.1 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index de64e79be..2a875fe95 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -44,7 +44,7 @@ object Dependencies { val momentJsVersion = "2.30.1" val seleniumVersion = "4.34.0" - val webDriverManagerVersion = "6.1.0" + val webDriverManagerVersion = "6.1.1" val scalaJsBenchmarkVersion = "0.10.0" val compilerPlugins = Def.setting(Seq( From 3048ed10da68b82d46fae2f0aba645c3187c2f89 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 15 Jul 2025 18:12:04 +0000 Subject: [PATCH 137/162] Update sbt-ide-settings to 1.1.3 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index aa4221171..17a05ddbd 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -6,7 +6,7 @@ libraryDependencies += "org.scala-js" %% "scalajs-env-jsdom-nodejs" % "1.1.0" addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.19.0") addSbtPlugin("org.scala-js" % "sbt-jsdependencies" % "1.0.2") addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.4") -addSbtPlugin("org.jetbrains.scala" % "sbt-ide-settings" % "1.1.2") +addSbtPlugin("org.jetbrains.scala" % "sbt-ide-settings" % "1.1.3") addSbtPlugin("com.github.sbt" % "sbt-less" % "2.0.1") addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.1") addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.11.1") From 517a9ddb280eb1362491515da38e3bd912098d7c Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 19 Jul 2025 18:34:43 +0000 Subject: [PATCH 138/162] Update webdrivermanager to 6.2.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 5b561cc91..3bb8d3e2a 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -44,7 +44,7 @@ object Dependencies { val momentJsVersion = "2.30.1" val seleniumVersion = "4.34.0" - val webDriverManagerVersion = "6.1.1" + val webDriverManagerVersion = "6.2.0" val scalaJsBenchmarkVersion = "0.10.0" val compilerPlugins = Def.setting(Seq( From 5a842bfc1092ca3bfb3f8d9793afb09ede6c49a9 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 23 Jul 2025 19:14:27 +0000 Subject: [PATCH 139/162] Update scalajs-dom to 2.8.1 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 5b561cc91..b06d43785 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -9,7 +9,7 @@ object Dependencies { val jqueryWrapperVersion = "3.3.0" - val scalaJsDomVersion = "2.8.0" + val scalaJsDomVersion = "2.8.1" val scalaTagsVersion = "0.13.1" val scalaCssVersion = "1.0.0" From de72b67f366a119a39b8675f6544054d18858086 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 12 Aug 2025 21:16:08 +0000 Subject: [PATCH 140/162] Update sbt, scripted-plugin to 1.11.4 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index b121e24fc..507add150 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ # suppress inspection "UnusedProperty" -sbt.version=1.11.3 +sbt.version=1.11.4 From 5af1e017f1eef72b3f24bdf4b618184ee09dfbf9 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 12 Aug 2025 21:16:11 +0000 Subject: [PATCH 141/162] Update selenium-java to 4.35.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 34b63f5f2..9a5c4b67b 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -43,7 +43,7 @@ object Dependencies { val bootstrap4DatepickerVersion = "5.39.0" val momentJsVersion = "2.30.1" - val seleniumVersion = "4.34.0" + val seleniumVersion = "4.35.0" val webDriverManagerVersion = "6.2.0" val scalaJsBenchmarkVersion = "0.10.0" From cb0ba45d49e09cff34aeabaad9ac4e516246d0fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Kozak?= Date: Thu, 21 Aug 2025 12:58:22 +0200 Subject: [PATCH 142/162] Update Jetty version to 12.1.0 and refactor dependency declarations for clarity --- project/Dependencies.scala | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 34b63f5f2..35f43e1cf 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -28,7 +28,7 @@ object Dependencies { val scalaLoggingVersion = "3.9.5" - val jettyVersion = "12.0.23" + val jettyVersion = "12.1.0" val typesafeConfigVersion = "1.4.4" val flexmarkVersion = "0.64.8" val logbackVersion = "1.3.15" @@ -98,8 +98,7 @@ object Dependencies { val rpcSjsDeps = rpcCrossDeps val rpcJsDeps = Def.setting(Seq( - "org.webjars" % "atmosphere-javascript" % atmosphereJSVersion / s"$atmosphereJSVersion/atmosphere.js" - minified s"$atmosphereJSVersion/atmosphere-min.js" + ("org.webjars" % "atmosphere-javascript" % atmosphereJSVersion / s"$atmosphereJSVersion/atmosphere.js").minified(s"$atmosphereJSVersion/atmosphere-min.js") )) private val restCrossDeps = Def.setting(Seq( @@ -143,11 +142,9 @@ object Dependencies { private val bootstrap4Resource = "js/bootstrap.bundle.js" val bootstrap4JsDeps = Def.setting(Seq[JSModuleID]( - "org.webjars" % "bootstrap" % bootstrap4Version / bootstrap4Resource - minified "js/bootstrap.bundle.min.js" dependsOn "jquery.js", - "org.webjars.npm" % "moment" % s"$momentJsVersion" / momentResource minified s"$momentJsVersion/min/moment.min.js", - "org.webjars" % "tempusdominus-bootstrap-4" % bootstrap4DatepickerVersion / "js/tempusdominus-bootstrap-4.js" - minified "js/tempusdominus-bootstrap-4.min.js" dependsOn(bootstrap4Resource, momentResource) + ("org.webjars" % "bootstrap" % bootstrap4Version / bootstrap4Resource).minified("js/bootstrap.bundle.min.js") dependsOn "jquery.js", + ("org.webjars.npm" % "moment" % momentJsVersion / momentResource).minified(s"$momentJsVersion/min/moment.min.js"), + ("org.webjars" % "tempusdominus-bootstrap-4" % bootstrap4DatepickerVersion / "js/tempusdominus-bootstrap-4.js").minified("js/tempusdominus-bootstrap-4.min.js") dependsOn(bootstrap4Resource, momentResource) )) val benchmarksSjsDeps = Def.setting(Seq( From 180ff3b1b154d8d2452f7c6e62fb15e699c45614 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sun, 24 Aug 2025 18:42:28 +0000 Subject: [PATCH 143/162] Update sbt-native-packager to 1.11.3 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 17a05ddbd..683fa8393 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -8,6 +8,6 @@ addSbtPlugin("org.scala-js" % "sbt-jsdependencies" % "1.0.2") addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.4") addSbtPlugin("org.jetbrains.scala" % "sbt-ide-settings" % "1.1.3") addSbtPlugin("com.github.sbt" % "sbt-less" % "2.0.1") -addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.1") +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.3") addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.11.1") addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.7") \ No newline at end of file From c5c70460863dbc8bbe5c5ea29d97a52ffe721011 Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Mon, 25 Aug 2025 09:51:37 +0200 Subject: [PATCH 144/162] Update sbt-ci-release to 1.11.2 (#1377) Co-authored-by: Dawid Dworak --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 683fa8393..ef5b2288f 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -9,5 +9,5 @@ addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.4") addSbtPlugin("org.jetbrains.scala" % "sbt-ide-settings" % "1.1.3") addSbtPlugin("com.github.sbt" % "sbt-less" % "2.0.1") addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.3") -addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.11.1") +addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.11.2") addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.7") \ No newline at end of file From d6c5179f23440f6811ba02d0de4fd201f8e22a3e Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Tue, 2 Sep 2025 12:23:44 +0200 Subject: [PATCH 145/162] Update webdrivermanager to 6.3.1 (#1382) --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 635f292dd..220986003 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -44,7 +44,7 @@ object Dependencies { val momentJsVersion = "2.30.1" val seleniumVersion = "4.35.0" - val webDriverManagerVersion = "6.2.0" + val webDriverManagerVersion = "6.3.1" val scalaJsBenchmarkVersion = "0.10.0" val compilerPlugins = Def.setting(Seq( From bd926b5285537adb5dd11e2e850af8765e711e7c Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Tue, 2 Sep 2025 12:23:57 +0200 Subject: [PATCH 146/162] Update upickle to 4.3.0 (#1381) --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 220986003..f73aa30e7 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -19,7 +19,7 @@ object Dependencies { val atmosphereJSVersion = "3.1.3" val atmosphereVersion = "2.7.15" - val upickleVersion = "4.2.1" // Tests only + val upickleVersion = "4.3.0" // Tests only val circeVersion = "0.14.14" // Tests only val circeDerivationVersion = "0.13.0-M5" // Tests only val monixVersion = "3.4.1" // udash-rest only From 6119e8282705e5fe1914d17ac0151d5aa4f7a0cb Mon Sep 17 00:00:00 2001 From: Scala Steward <43047562+scala-steward@users.noreply.github.com> Date: Tue, 2 Sep 2025 12:24:27 +0200 Subject: [PATCH 147/162] Update sbt, scripted-plugin to 1.11.5 (#1380) --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index 507add150..5adec42fa 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ # suppress inspection "UnusedProperty" -sbt.version=1.11.4 +sbt.version=1.11.5 From 727f4ef61e4d45c04e2e5f8587134ebbbe99653f Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sun, 7 Sep 2025 21:49:36 +0000 Subject: [PATCH 148/162] Update sbt-scalajs, scalajs-compiler, ... to 1.20.1 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index ef5b2288f..b794fe257 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -3,7 +3,7 @@ logLevel := Level.Warn libraryDependencies += "org.scala-js" %% "scalajs-env-selenium" % "1.1.1" libraryDependencies += "org.scala-js" %% "scalajs-env-jsdom-nodejs" % "1.1.0" -addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.19.0") +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.20.1") addSbtPlugin("org.scala-js" % "sbt-jsdependencies" % "1.0.2") addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.4") addSbtPlugin("org.jetbrains.scala" % "sbt-ide-settings" % "1.1.3") From 8afbd3be9072e5eac21a97d1f720fa4def3b6660 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sun, 7 Sep 2025 21:49:38 +0000 Subject: [PATCH 149/162] Update sbt, scripted-plugin to 1.11.6 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index 5adec42fa..a083f147a 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ # suppress inspection "UnusedProperty" -sbt.version=1.11.5 +sbt.version=1.11.6 From ec4da05375356654413095b714fda20224930293 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 9 Sep 2025 20:57:14 +0000 Subject: [PATCH 150/162] Update jetty-client to 12.1.1 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index f73aa30e7..b7cc80c42 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -28,7 +28,7 @@ object Dependencies { val scalaLoggingVersion = "3.9.5" - val jettyVersion = "12.1.0" + val jettyVersion = "12.1.1" val typesafeConfigVersion = "1.4.4" val flexmarkVersion = "0.64.8" val logbackVersion = "1.3.15" From 22fdce2b29da10ea3a49e18e5dc91a3f5e988219 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Thu, 11 Sep 2025 22:39:13 +0000 Subject: [PATCH 151/162] Update upickle to 4.3.2 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index f73aa30e7..9d87343c4 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -19,7 +19,7 @@ object Dependencies { val atmosphereJSVersion = "3.1.3" val atmosphereVersion = "2.7.15" - val upickleVersion = "4.3.0" // Tests only + val upickleVersion = "4.3.2" // Tests only val circeVersion = "0.14.14" // Tests only val circeDerivationVersion = "0.13.0-M5" // Tests only val monixVersion = "3.4.1" // udash-rest only From aadfded8e3047349b49379bfca521423b1b67d98 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Thu, 11 Sep 2025 22:39:15 +0000 Subject: [PATCH 152/162] Update typesafe:config to 1.4.5 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index f73aa30e7..062eae740 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -29,7 +29,7 @@ object Dependencies { val scalaLoggingVersion = "3.9.5" val jettyVersion = "12.1.0" - val typesafeConfigVersion = "1.4.4" + val typesafeConfigVersion = "1.4.5" val flexmarkVersion = "0.64.8" val logbackVersion = "1.3.15" val janinoVersion = "3.1.12" From fbd44ccb71403d811a9d8f59abfd89c123e0cbda Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 13 Sep 2025 20:22:59 +0000 Subject: [PATCH 153/162] Update webdrivermanager to 6.3.2 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index f73aa30e7..1f21a2227 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -44,7 +44,7 @@ object Dependencies { val momentJsVersion = "2.30.1" val seleniumVersion = "4.35.0" - val webDriverManagerVersion = "6.3.1" + val webDriverManagerVersion = "6.3.2" val scalaJsBenchmarkVersion = "0.10.0" val compilerPlugins = Def.setting(Seq( From 085736849972431ad4684406a79fde9ba1b8b0e9 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Mon, 15 Sep 2025 18:12:01 +0000 Subject: [PATCH 154/162] Update scala-logging to 3.9.6 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index f73aa30e7..f3d5e949c 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -26,7 +26,7 @@ object Dependencies { val sttpVersion = "3.11.0" - val scalaLoggingVersion = "3.9.5" + val scalaLoggingVersion = "3.9.6" val jettyVersion = "12.1.0" val typesafeConfigVersion = "1.4.4" From b742878e8be3312b544c9bb7433217b557e1055b Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Mon, 22 Sep 2025 17:49:51 +0200 Subject: [PATCH 155/162] Switch Jetty from legacy GzipHandler to CompressionHandler --- .../main/scala/io/udash/web/server/ApplicationServer.scala | 6 +++--- project/Dependencies.scala | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/guide/backend/src/main/scala/io/udash/web/server/ApplicationServer.scala b/guide/backend/src/main/scala/io/udash/web/server/ApplicationServer.scala index 645d2e20d..db1b1f02d 100644 --- a/guide/backend/src/main/scala/io/udash/web/server/ApplicationServer.scala +++ b/guide/backend/src/main/scala/io/udash/web/server/ApplicationServer.scala @@ -10,13 +10,13 @@ import io.udash.web.guide.rest.ExposedRestInterfaces import io.udash.web.guide.rpc.ExposedRpcInterfaces import io.udash.web.guide.{GuideExceptions, MainServerRPC} import monix.execution.Scheduler +import org.eclipse.jetty.compression.server.CompressionHandler import org.eclipse.jetty.ee8.nested.SessionHandler import org.eclipse.jetty.ee8.servlet.{DefaultServlet, ServletContextHandler, ServletHolder} import org.eclipse.jetty.ee8.websocket.javax.server.config.JavaxWebSocketServletContainerInitializer import org.eclipse.jetty.rewrite.handler.{RewriteHandler, RewriteRegexRule} import org.eclipse.jetty.server.Server import org.eclipse.jetty.server.handler.ContextHandlerCollection -import org.eclipse.jetty.server.handler.gzip.GzipHandler import org.eclipse.jetty.util.resource.ResourceFactory import java.nio.file.Path @@ -31,7 +31,7 @@ class ApplicationServer(val port: Int, homepageResourceBase: String, guideResour server.stop() private val homepage = - new GzipHandler(createContextHandler( + new CompressionHandler(createContextHandler( hosts = Array("udash.io", "www.udash.io", "udash.local", "127.0.0.1"), resourceBase = homepageResourceBase ).get()) @@ -64,7 +64,7 @@ class ApplicationServer(val port: Int, homepageResourceBase: String, guideResour contextHandler.addServlet(new ServletHolder(RestServlet[MainServerREST](new ExposedRestInterfaces)), "/rest_api/*") - new GzipHandler(contextHandler.get()) + new CompressionHandler(contextHandler.get()) } server.setHandler( diff --git a/project/Dependencies.scala b/project/Dependencies.scala index b7cc80c42..840bf3610 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -162,6 +162,8 @@ object Dependencies { "org.eclipse.jetty" % "jetty-rewrite" % jettyVersion, "org.eclipse.jetty.ee8.websocket" % "jetty-ee8-websocket-javax-server" % jettyVersion, + "org.eclipse.jetty.compression" % "jetty-compression-server" % jettyVersion, + "org.eclipse.jetty.compression" % "jetty-compression-gzip" % jettyVersion, "com.typesafe" % "config" % typesafeConfigVersion, From 08ce7a501fb6092c97fb26af3c7eadac7ba5f10e Mon Sep 17 00:00:00 2001 From: Dawid Dworak Date: Mon, 22 Sep 2025 18:09:56 +0200 Subject: [PATCH 156/162] Remove compression in EndpointsIntegrationTest Not needed for REST testing, avoids redundant Jetty dependencies --- .../test/scala/io/udash/rest/EndpointsIntegrationTest.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rest/.jvm/src/test/scala/io/udash/rest/EndpointsIntegrationTest.scala b/rest/.jvm/src/test/scala/io/udash/rest/EndpointsIntegrationTest.scala index df4e3d5a4..20c588e2e 100644 --- a/rest/.jvm/src/test/scala/io/udash/rest/EndpointsIntegrationTest.scala +++ b/rest/.jvm/src/test/scala/io/udash/rest/EndpointsIntegrationTest.scala @@ -7,7 +7,6 @@ import monix.execution.Scheduler import org.eclipse.jetty.ee8.nested.SessionHandler import org.eclipse.jetty.ee8.servlet.{ServletContextHandler, ServletHolder} import org.eclipse.jetty.server.Server -import org.eclipse.jetty.server.handler.gzip.GzipHandler import org.scalatest.BeforeAndAfterAll import org.scalatest.concurrent.{Eventually, ScalaFutures} import org.scalatest.time.{Millis, Seconds, Span} @@ -31,7 +30,7 @@ class EndpointsIntegrationTest extends UdashSharedTest with UsesHttpServer with contextHandler.setSessionHandler(new SessionHandler) contextHandler.addServlet(holder, s"$contextPrefix/*") - server.setHandler(new GzipHandler(contextHandler.get())) + server.setHandler(contextHandler.get()) } def futureHandle(rawHandle: RawRest.HandleRequest): RestRequest => Future[RestResponse] = From 13e4cc8fe67cf5c69aceefa1d4aead88e700d20b Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Mon, 29 Sep 2025 17:34:45 +0000 Subject: [PATCH 157/162] Update sbt-jmh to 0.4.8 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index b794fe257..01868ef76 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -10,4 +10,4 @@ addSbtPlugin("org.jetbrains.scala" % "sbt-ide-settings" % "1.1.3") addSbtPlugin("com.github.sbt" % "sbt-less" % "2.0.1") addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.3") addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.11.2") -addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.7") \ No newline at end of file +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.8") \ No newline at end of file From 019508e21155f2cffee31f1985166f966e506fec Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 1 Oct 2025 20:16:33 +0000 Subject: [PATCH 158/162] Update commons-analyzer, commons-core, ... to 2.24.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index ad820e55e..7b13d722d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -14,7 +14,7 @@ object Dependencies { val scalaCssVersion = "1.0.0" val servletVersion = "4.0.1" - val avsCommonsVersion = "2.23.1" + val avsCommonsVersion = "2.24.0" val atmosphereJSVersion = "3.1.3" val atmosphereVersion = "2.7.15" From 08fead786fc769842170c7925cf9a096d4fd8d2b Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 1 Oct 2025 20:16:46 +0000 Subject: [PATCH 159/162] Update sbt-native-packager to 1.11.4 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 01868ef76..fce83de23 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -8,6 +8,6 @@ addSbtPlugin("org.scala-js" % "sbt-jsdependencies" % "1.0.2") addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.4") addSbtPlugin("org.jetbrains.scala" % "sbt-ide-settings" % "1.1.3") addSbtPlugin("com.github.sbt" % "sbt-less" % "2.0.1") -addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.3") +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.4") addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.11.2") addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.8") \ No newline at end of file From b04eb447be4ef480b9a17c2eb746e8fab45450b9 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Wed, 1 Oct 2025 20:16:50 +0000 Subject: [PATCH 160/162] Update circe-core, circe-parser to 0.14.15 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index ad820e55e..879720811 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -20,7 +20,7 @@ object Dependencies { val atmosphereVersion = "2.7.15" val upickleVersion = "4.3.2" // Tests only - val circeVersion = "0.14.14" // Tests only + val circeVersion = "0.14.15" // Tests only val circeDerivationVersion = "0.13.0-M5" // Tests only val monixVersion = "3.4.1" // udash-rest only From 3e166e8566cd02620783081494cbca57890a56f6 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Fri, 3 Oct 2025 19:17:23 +0000 Subject: [PATCH 161/162] Update selenium-java to 4.36.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index ad820e55e..4b58d2837 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -43,7 +43,7 @@ object Dependencies { val bootstrap4DatepickerVersion = "5.39.0" val momentJsVersion = "2.30.1" - val seleniumVersion = "4.35.0" + val seleniumVersion = "4.36.0" val webDriverManagerVersion = "6.3.2" val scalaJsBenchmarkVersion = "0.10.0" From ae5cd25b6b8a1f7f208f2ceb811ed64a13219a31 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Mon, 6 Oct 2025 08:05:18 +0000 Subject: [PATCH 162/162] Update sbt, scripted-plugin to 1.11.7 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index a083f147a..8e6360838 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ # suppress inspection "UnusedProperty" -sbt.version=1.11.6 +sbt.version=1.11.7