clear require cache only when needed (#56198)

### What?

clear require cache only when there has been changes

Before we cleared the require.cache after every ensurePage call. This is
too much. The new approach compares the hashes of all emitted files with
the previous hashes and only clears require.cache when they differ.

### Why?

reloading a page and client navigation is slow due the re-requiring
server files


Closes WEB-1686

Author: sokra

packages/next-swc/crates/napi/src/next_api/endpoint.rs

@@ -1,7 +1,10 @@
 use std::ops::Deref;
 
 use napi::{bindgen_prelude::External, JsFunction};
-use next_api::route::{Endpoint, WrittenEndpoint};
+use next_api::{
+    route::{Endpoint, WrittenEndpoint},
+    server_paths::ServerPath,
+};
 use turbo_tasks::Vc;
 use turbopack_binding::turbopack::core::error::PrettyPrintError;
 
@@ -14,12 +17,28 @@ use super::utils::{
 #[derive(Default)]
 pub struct NapiEndpointConfig {}
 
+#[napi(object)]
+#[derive(Default)]
+pub struct NapiServerPath {
+    pub path: String,
+    pub content_hash: String,
+}
+
+impl From<&ServerPath> for NapiServerPath {
+    fn from(server_path: &ServerPath) -> Self {
+        Self {
+            path: server_path.path.clone(),
+            content_hash: format!("{:x}", server_path.content_hash),
+        }
+    }
+}
+
 #[napi(object)]
 #[derive(Default)]
 pub struct NapiWrittenEndpoint {
     pub r#type: String,
     pub entry_path: Option<String>,
-    pub server_paths: Option<Vec<String>>,
+    pub server_paths: Option<Vec<NapiServerPath>>,
     pub files: Option<Vec<String>>,
     pub global_var_name: Option<String>,
     pub config: NapiEndpointConfig,
@@ -34,7 +53,7 @@ impl From<&WrittenEndpoint> for NapiWrittenEndpoint {
             } => Self {
                 r#type: "nodejs".to_string(),
                 entry_path: Some(server_entry_path.clone()),
-                server_paths: Some(server_paths.clone()),
+                server_paths: Some(server_paths.iter().map(From::from).collect()),
                 ..Default::default()
             },
             WrittenEndpoint::Edge {
@@ -44,7 +63,7 @@ impl From<&WrittenEndpoint> for NapiWrittenEndpoint {
             } => Self {
                 r#type: "edge".to_string(),
                 files: Some(files.clone()),
-                server_paths: Some(server_paths.clone()),
+                server_paths: Some(server_paths.iter().map(From::from).collect()),
                 global_var_name: Some(global_var_name.clone()),
                 ..Default::default()
             },

packages/next-swc/crates/next-api/src/app.rs

@@ -1,6 +1,5 @@
 use anyhow::{bail, Context, Result};
 use next_core::{
-    all_server_paths,
     app_structure::{
         get_entrypoints, Entrypoint as AppEntrypoint, Entrypoints as AppEntrypoints, LoaderTree,
         MetadataItem,
@@ -55,6 +54,7 @@ use turbopack_binding::{
 use crate::{
     project::Project,
     route::{Endpoint, Route, Routes, WrittenEndpoint},
+    server_paths::all_server_paths,
 };
 
 #[turbo_tasks::value]

packages/next-swc/crates/next-api/src/lib.rs

@@ -8,6 +8,7 @@ mod middleware;
 mod pages;
 pub mod project;
 pub mod route;
+pub mod server_paths;
 mod versioned_content_map;
 
 // Declare build-time information variables generated in build.rs

packages/next-swc/crates/next-api/src/middleware.rs

@@ -1,6 +1,5 @@
 use anyhow::{bail, Context, Result};
 use next_core::{
-    all_server_paths,
     middleware::get_middleware_module,
     mode::NextMode,
     next_edge::entry::wrap_edge_entry,
@@ -27,6 +26,7 @@ use turbopack_binding::{
 use crate::{
     project::Project,
     route::{Endpoint, WrittenEndpoint},
+    server_paths::all_server_paths,
 };
 
 #[turbo_tasks::value]

packages/next-swc/crates/next-api/src/pages.rs

@@ -1,7 +1,7 @@
 use anyhow::{bail, Context, Result};
 use indexmap::IndexMap;
 use next_core::{
-    all_server_paths, create_page_loader_entry_module, get_asset_path_from_pathname,
+    create_page_loader_entry_module, get_asset_path_from_pathname,
     get_edge_resolve_options_context,
     mode::NextMode,
     next_client::{
@@ -64,6 +64,7 @@ use turbopack_binding::{
 use crate::{
     project::Project,
     route::{Endpoint, Route, Routes, WrittenEndpoint},
+    server_paths::all_server_paths,
 };
 
 #[turbo_tasks::value]

packages/next-swc/crates/next-api/src/route.rs

@@ -1,6 +1,8 @@
 use indexmap::IndexMap;
 use turbo_tasks::{Completion, Vc};
 
+use crate::server_paths::ServerPath;
+
 #[turbo_tasks::value(shared)]
 #[derive(Copy, Clone, Debug)]
 pub enum Route {
@@ -34,15 +36,13 @@ pub enum WrittenEndpoint {
     NodeJs {
         /// Relative to the root_path
         server_entry_path: String,
-        /// Relative to the root_path
-        server_paths: Vec<String>,
+        server_paths: Vec<ServerPath>,
     },
     Edge {
         /// Relative to the root_path
         files: Vec<String>,
         global_var_name: String,
-        /// Relative to the root_path
-        server_paths: Vec<String>,
+        server_paths: Vec<ServerPath>,
     },
 }
 

packages/next-swc/crates/next-api/src/server_paths.rs

@@ -0,0 +1,57 @@
+use anyhow::Result;
+use next_core::all_assets_from_entries;
+use serde::{Deserialize, Serialize};
+use turbo_tasks::{trace::TraceRawVcs, TryFlatJoinIterExt, Vc};
+use turbopack_binding::{
+    turbo::tasks_fs::FileSystemPath,
+    turbopack::core::{
+        asset::{Asset, AssetContent},
+        output::{OutputAsset, OutputAssets},
+    },
+};
+
+/// A reference to a server file with content hash for change detection
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, TraceRawVcs)]
+pub struct ServerPath {
+    /// Relative to the root_path
+    pub path: String,
+    pub content_hash: u64,
+}
+
+/// A list of server paths
+#[turbo_tasks::value(transparent)]
+pub struct ServerPaths(Vec<ServerPath>);
+
+/// Return a list of all server paths with filename and hash for all output
+/// assets references from the `assets` list. Server paths are identified by
+/// being inside of `node_root`.
+#[turbo_tasks::function]
+pub async fn all_server_paths(
+    assets: Vc<OutputAssets>,
+    node_root: Vc<FileSystemPath>,
+) -> Result<Vc<ServerPaths>> {
+    let all_assets = all_assets_from_entries(assets).await?;
+    let node_root = &node_root.await?;
+    Ok(Vc::cell(
+        all_assets
+            .iter()
+            .map(|&asset| async move {
+                Ok(
+                    if let Some(path) = node_root.get_path_to(&*asset.ident().path().await?) {
+                        let content_hash = match *asset.content().await? {
+                            AssetContent::File(file) => *file.hash().await?,
+                            AssetContent::Redirect { .. } => 0,
+                        };
+                        Some(ServerPath {
+                            path: path.to_string(),
+                            content_hash,
+                        })
+                    } else {
+                        None
+                    },
+                )
+            })
+            .try_flat_join()
+            .await?,
+    ))
+}

packages/next-swc/crates/next-core/src/emit.rs

@@ -1,37 +1,14 @@
 use anyhow::Result;
 use turbo_tasks::{
     graph::{AdjacencyMap, GraphTraversal},
-    Completion, Completions, TryFlatJoinIterExt, TryJoinIterExt, Vc,
+    Completion, Completions, TryFlatJoinIterExt, Vc,
 };
 use turbo_tasks_fs::{rebase, FileSystemPath};
 use turbopack_binding::turbopack::core::{
     asset::Asset,
     output::{OutputAsset, OutputAssets},
 };
 
-#[turbo_tasks::function]
-pub async fn all_server_paths(
-    assets: Vc<OutputAssets>,
-    node_root: Vc<FileSystemPath>,
-) -> Result<Vc<Vec<String>>> {
-    let all_assets = all_assets_from_entries(assets).await?;
-    let node_root = &node_root.await?;
-    Ok(Vc::cell(
-        all_assets
-            .iter()
-            .map(|&asset| async move {
-                Ok(node_root
-                    .get_path_to(&*asset.ident().path().await?)
-                    .map(|s| s.to_string()))
-            })
-            .try_join()
-            .await?
-            .into_iter()
-            .flatten()
-            .collect(),
-    ))
-}
-
 /// Emits all assets transitively reachable from the given chunks, that are
 /// inside the node root or the client root.
 ///

packages/next-swc/crates/next-core/src/lib.rs

@@ -46,9 +46,7 @@ pub mod util;
 pub use app_segment_config::{
     parse_segment_config_from_loader_tree, parse_segment_config_from_source,
 };
-pub use emit::{
-    all_assets_from_entries, all_server_paths, emit_all_assets, emit_assets, emit_client_assets,
-};
+pub use emit::{all_assets_from_entries, emit_all_assets, emit_assets, emit_client_assets};
 pub use next_edge::context::{
     get_edge_chunking_context, get_edge_compile_time_info, get_edge_resolve_options_context,
 };

packages/next/src/build/swc/index.ts

@@ -540,20 +540,25 @@ interface EndpointConfig {
   preferredRegion?: string
 }
 
+export type ServerPath = {
+  path: string
+  contentHash: string
+}
+
 export type WrittenEndpoint =
   | {
       type: 'nodejs'
       /** The entry path for the endpoint. */
       entryPath: string
       /** All server paths that has been written for the endpoint. */
-      serverPaths: string[]
+      serverPaths: ServerPath[]
       config: EndpointConfig
     }
   | {
       type: 'edge'
       files: string[]
       /** All server paths that has been written for the endpoint. */
-      serverPaths: string[]
+      serverPaths: ServerPath[]
       globalVarName: string
       config: EndpointConfig
     }