improve changelog

docs/victoriametrics/changelog/CHANGELOG.md

@@ -25,6 +25,7 @@ The sandbox cluster installation runs under the constant load generated by
 See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/).
 
 ## tip
+* FEATURE: record all logs in `vm_log_messages_total` regardless of the log level set via `-loggerLevel`. This will be helpful in troubleshooting. See [#10304](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10304)
 
 ## [v1.134.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.134.0)
 

lib/logger/logger.go

@@ -148,11 +148,20 @@ func logLevel(level, format string, args []any) {
 }
 
 func logLevelSkipframes(skipframes int, level, format string, args []any) {
+	file, line := logLocation(3 + skipframes)
+	location := fmt.Sprintf("%s:%d", file, line)
+	isPrinted := true
+	defer func() {
+		// Increase vm_log_messages_total
+		counterName := fmt.Sprintf(`vm_log_messages_total{app_version=%q, level=%q, location=%q, is_printed="%t"}`, buildinfo.Version, strings.ToLower(level), location, isPrinted)
+		metrics.GetOrCreateCounter(counterName).Inc()
+	}()
 	if shouldSkipLog(level) {
+		isPrinted = false
 		return
 	}
 	msg := formatLogMessage(*maxLogArgLen, format, args)
-	logMessage(level, msg, 3+skipframes)
+	isPrinted = logMessage(level, msg, location)
 }
 
 func formatLogMessage(maxArgLen int, format string, args []any) string {
@@ -172,6 +181,19 @@ func formatLogMessage(maxArgLen int, format string, args []any) string {
 	return fmt.Sprintf(format, args...)
 }
 
+func logLocation(skipframes int) (string, int) {
+	_, file, line, ok := runtime.Caller(skipframes)
+	if !ok {
+		file = "???"
+		line = 0
+	}
+	if n := strings.Index(file, "/VictoriaMetrics/"); n >= 0 {
+		// Strip /VictoriaMetrics/ prefix
+		file = file[n+len("/VictoriaMetrics/"):]
+	}
+	return file, line
+}
+
 func logLimiterCleaner() {
 	for {
 		time.Sleep(time.Second)
@@ -235,22 +257,12 @@ func (lw *logWriter) Write(p []byte) (int, error) {
 	return len(p), nil
 }
 
-func logMessage(level, msg string, skipframes int) {
+func logMessage(level, msg string, location string) bool {
 	timestamp := ""
 	if !*disableTimestamps {
 		timestamp = time.Now().In(timezone).Format("2006-01-02T15:04:05.000Z0700")
 	}
 	levelLowercase := strings.ToLower(level)
-	_, file, line, ok := runtime.Caller(skipframes)
-	if !ok {
-		file = "???"
-		line = 0
-	}
-	if n := strings.Index(file, "/VictoriaMetrics/"); n >= 0 {
-		// Strip /VictoriaMetrics/ prefix
-		file = file[n+len("/VictoriaMetrics/"):]
-	}
-	location := fmt.Sprintf("%s:%d", file, line)
 
 	// rate limit ERROR and WARN log messages with given limit.
 	if level == "ERROR" || level == "WARN" {
@@ -260,7 +272,7 @@ func logMessage(level, msg string, skipframes int) {
 		}
 		ok, suppressMessage := logLimiter.needSuppress(location, limit)
 		if ok {
-			return
+			return false
 		}
 		if len(suppressMessage) > 0 {
 			msg = suppressMessage + msg
@@ -302,10 +314,6 @@ func logMessage(level, msg string, skipframes int) {
 	fmt.Fprint(output, logMsg)
 	mu.Unlock()
 
-	// Increment vm_log_messages_total
-	counterName := fmt.Sprintf(`vm_log_messages_total{app_version=%q, level=%q, location=%q}`, buildinfo.Version, levelLowercase, location)
-	metrics.GetOrCreateCounter(counterName).Inc()
-
 	switch level {
 	case "PANIC":
 		if *loggerFormat == "json" {
@@ -316,6 +324,7 @@ func logMessage(level, msg string, skipframes int) {
 	case "FATAL":
 		os.Exit(-1)
 	}
+	return true
 }
 
 var mu sync.Mutex