perf: Optimize website log reading (1Panel-dev#9755)

Refs 1Panel-dev#9383

Author: zhengkunwang223

agent/app/dto/response/file.go

@@ -38,12 +38,12 @@ type FileWgetRes struct {
 }
 
 type FileLineContent struct {
-	Content    string   `json:"content"`
 	End        bool     `json:"end"`
 	Path       string   `json:"path"`
 	Total      int      `json:"total"`
 	TaskStatus string   `json:"taskStatus"`
 	Lines      []string `json:"lines"`
+	Scope      string   `json:"scope"`
 }
 
 type FileExist struct {

agent/app/service/file.go

@@ -542,24 +542,48 @@ func (f *FileService) ReadLogByLine(req request.FileReadByLineReq) (*response.Fi
 		logFilePath = path.Join(global.Dir.DataDir, fmt.Sprintf("apps/mariadb/%s/db/data/1Panel-slow.log", req.Name))
 	}
 
-	lines, isEndOfFile, total, err := files.ReadFileByLine(logFilePath, req.Page, req.PageSize, req.Latest)
+	file, err := os.Open(logFilePath)
 	if err != nil {
 		return nil, err
 	}
-	if req.Latest && req.Page == 1 && len(lines) < 1000 && total > 1 {
-		preLines, _, _, err := files.ReadFileByLine(logFilePath, total-1, req.PageSize, false)
+	defer file.Close()
+	stat, err := file.Stat()
+	if err != nil {
+		return nil, err
+	}
+	var (
+		lines       []string
+		isEndOfFile bool
+		total       int
+		scope       string
+	)
+
+	if stat.Size() > 500*1024*1024 {
+		lines, err = files.TailFromEnd(logFilePath, req.PageSize)
+		isEndOfFile = true
+		scope = "tail"
+	} else {
+		lines, isEndOfFile, total, err = files.ReadFileByLine(logFilePath, req.Page, req.PageSize, req.Latest)
 		if err != nil {
 			return nil, err
 		}
-		lines = append(preLines, lines...)
+		if req.Latest && req.Page == 1 && len(lines) < 1000 && total > 1 {
+			preLines, _, _, err := files.ReadFileByLine(logFilePath, total-1, req.PageSize, false)
+			if err != nil {
+				return nil, err
+			}
+			lines = append(preLines, lines...)
+		}
+		scope = "page"
 	}
+
 	res := &response.FileLineContent{
-		Content:    strings.Join(lines, "\n"),
 		End:        isEndOfFile,
 		Path:       logFilePath,
 		Total:      total,
 		TaskStatus: taskStatus,
 		Lines:      lines,
+		Scope:      scope,
 	}
 	return res, nil
 }

agent/utils/files/utils.go

@@ -101,6 +101,66 @@ func countLines(path string) (int, error) {
 	}
 }
 
+func TailFromEnd(filename string, lines int) ([]string, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	stat, err := file.Stat()
+	if err != nil {
+		return nil, err
+	}
+	fileSize := stat.Size()
+
+	bufSize := int64(4096)
+	var result []string
+	var leftover string
+
+	for offset := fileSize; offset > 0 && len(result) < lines; {
+		readSize := bufSize
+		if offset < bufSize {
+			readSize = offset
+		}
+		offset -= readSize
+
+		buf := make([]byte, readSize)
+		_, err := file.ReadAt(buf, offset)
+		if err != nil && err != io.EOF {
+			return nil, err
+		}
+
+		data := string(buf) + leftover
+		linesInChunk := strings.Split(data, "\n")
+
+		if offset > 0 {
+			leftover = linesInChunk[0]
+			linesInChunk = linesInChunk[1:]
+		} else {
+			leftover = ""
+		}
+
+		for i := len(linesInChunk) - 1; i >= 0; i-- {
+			if len(result) < lines {
+				if !(i == len(linesInChunk)-1 && linesInChunk[i] == "" && len(result) == 0) {
+					result = append([]string{linesInChunk[i]}, result...)
+				}
+			}
+		}
+	}
+
+	if leftover != "" && len(result) < lines {
+		result = append([]string{leftover}, result...)
+	}
+
+	if len(result) > lines {
+		result = result[len(result)-lines:]
+	}
+
+	return result, nil
+}
+
 func ReadFileByLine(filename string, page, pageSize int, latest bool) (lines []string, isEndOfFile bool, total int, err error) {
 	if !NewFileOp().Stat(filename) {
 		return

frontend/src/components/log/file/index.vue

@@ -1,6 +1,7 @@
 <template>
     <div v-loading="firstLoading">
         <div v-if="defaultButton">
+            <el-button icon="Refresh" @click="getContent(false)">{{ $t('commons.button.refresh') }}</el-button>
             <el-checkbox
                 border
                 :disabled="isTailDisabled"
@@ -133,11 +134,10 @@ const end = ref(false);
 const lastLogs = ref([]);
 const maxPage = ref(0);
 const minPage = ref(0);
-let timer: NodeJS.Timer | null = null;
+let timer: ReturnType<typeof setInterval> | null = null;
 const logPath = ref('');
-
+const showTail = ref(false);
 const isTailDisabled = ref();
-
 const firstLoading = ref(false);
 const logs = ref<string[]>([]);
 const logContainer = ref<HTMLElement | null>(null);
@@ -222,6 +222,10 @@ const getContent = async (pre: boolean) => {
         firstLoading.value = false;
     }
 
+    if (res.data.scope == 'tail') {
+        showTail.value = false;
+    }
+
     if (res.data.taskStatus && res.data.taskStatus !== 'Executing') {
         isTailDisabled.value = true;
     }
@@ -342,6 +346,7 @@ const containerStyle = computed(() => ({
 }));
 
 onMounted(async () => {
+    showTail.value = props.showTail;
     logs.value = [];
     isTailDisabled.value = false;
     firstLoading.value = true;