util: add support columnar scan details (#69331)

DEPS.bzl

@@ -6582,13 +6582,13 @@ def go_deps():
         name = "com_github_pingcap_tipb",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/pingcap/tipb",
-        sha256 = "2d6aaef873e175599c39f6fe3cf85cabce88a69c157ff7ec4c22fbcbc97648dd",
-        strip_prefix = "github.com/pingcap/tipb@v0.0.0-20260210113932-1447c9d7e9fe",
+        sha256 = "5a8c6a6e4487592fea5550c2a550826bd0ad7025c7bbec12365ecc7aed81db0d",
+        strip_prefix = "github.com/pingcap/tipb@v0.0.0-20260617071407-7c071244534b",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260210113932-1447c9d7e9fe.zip",
-            "http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260210113932-1447c9d7e9fe.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260210113932-1447c9d7e9fe.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260210113932-1447c9d7e9fe.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260617071407-7c071244534b.zip",
+            "http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260617071407-7c071244534b.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260617071407-7c071244534b.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260617071407-7c071244534b.zip",
         ],
     )
     go_repository(

go.mod

@@ -106,7 +106,7 @@ require (
 	github.com/pingcap/metering_sdk v0.0.0-20260324055927-14fead745f1d
 	github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5
 	github.com/pingcap/tidb/pkg/parser v0.0.0-20211011031125-9b13dc409c5e
-	github.com/pingcap/tipb v0.0.0-20260210113932-1447c9d7e9fe
+	github.com/pingcap/tipb v0.0.0-20260617071407-7c071244534b
 	github.com/prometheus/client_golang v1.23.0
 	github.com/prometheus/client_model v0.6.2
 	github.com/prometheus/common v0.65.0

go.sum

@@ -738,8 +738,8 @@ github.com/pingcap/metering_sdk v0.0.0-20260324055927-14fead745f1d h1:5JCgncG9X7
 github.com/pingcap/metering_sdk v0.0.0-20260324055927-14fead745f1d/go.mod h1:HMNxmg0/lrn3SPGJ6LTZqP0WwEpcXMu9s/4TWJbzT8w=
 github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 h1:T4pXRhBflzDeAhmOQHNPRRogMYxP13V7BkYw3ZsoSfE=
 github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5/go.mod h1:rlimy0GcTvjiJqvD5mXTRr8O2eNZPBrcUgiWVYp9530=
-github.com/pingcap/tipb v0.0.0-20260210113932-1447c9d7e9fe h1:Zmz9mON+2NoKDVjkJbk6NZbFoTzVzk8MPTbRnu+MiVM=
-github.com/pingcap/tipb v0.0.0-20260210113932-1447c9d7e9fe/go.mod h1:RM8iRcMalzOthG2XJxnNBniM4xFGb/lDwHUwqkaVzt4=
+github.com/pingcap/tipb v0.0.0-20260617071407-7c071244534b h1:10OHx2/2noFfPU1K3c3QXZshsw3XvKssk/mjbLdw8DI=
+github.com/pingcap/tipb v0.0.0-20260617071407-7c071244534b/go.mod h1:RM8iRcMalzOthG2XJxnNBniM4xFGb/lDwHUwqkaVzt4=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=

pkg/util/execdetails/execdetails_test.go

@@ -321,6 +321,38 @@ func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, Nu
 		NumIterations: &NumIterations, Concurrency: &Concurrency, ExecutorId: &ExecutorID, DetailInfo: &tipb.ExecutorExecutionSummary_TiflashScanContext{TiflashScanContext: &tiflashScanContext}, TiflashWaitSummary: &tiflashWaitSummary, TiflashNetworkSummary: &tiflashNetworkSummary}
 }
 
+func mockExecutorExecutionSummaryForTiFlashColumnar(TimeProcessedNs, NumProducedRows, NumIterations, Concurrency, regions, readTasks, physicalTables, columns, userReadBytes, mvccInputRows, mvccInputBytes, mvccOutputRows, totalReadBlockMs, totalSerializeBlockMs, totalInitReaderMs, totalPrefetchMs, roughCheckTotalPacks, roughCheckSelectedPacks, roughCheckSkippedPacks, roughCheckUnknownPacks, remoteSegments, totalSegments, totalDeserializeBlockMs uint64, ExecutorID string) *tipb.ExecutorExecutionSummary {
+	columnarScanContext := tipb.ColumnarScanContext{
+		Regions:                 &regions,
+		ReadTasks:               &readTasks,
+		PhysicalTables:          &physicalTables,
+		Columns:                 &columns,
+		UserReadBytes:           &userReadBytes,
+		MvccInputRows:           &mvccInputRows,
+		MvccInputBytes:          &mvccInputBytes,
+		MvccOutputRows:          &mvccOutputRows,
+		TotalReadBlockMs:        &totalReadBlockMs,
+		TotalSerializeBlockMs:   &totalSerializeBlockMs,
+		TotalInitReaderMs:       &totalInitReaderMs,
+		TotalPrefetchMs:         &totalPrefetchMs,
+		RoughCheckTotalPacks:    &roughCheckTotalPacks,
+		RoughCheckSelectedPacks: &roughCheckSelectedPacks,
+		RoughCheckSkippedPacks:  &roughCheckSkippedPacks,
+		RoughCheckUnknownPacks:  &roughCheckUnknownPacks,
+		RemoteSegments:          &remoteSegments,
+		TotalSegments:           &totalSegments,
+		TotalDeserializeBlockMs: &totalDeserializeBlockMs,
+	}
+	return &tipb.ExecutorExecutionSummary{
+		TimeProcessedNs: &TimeProcessedNs,
+		NumProducedRows: &NumProducedRows,
+		NumIterations:   &NumIterations,
+		Concurrency:     &Concurrency,
+		ExecutorId:      &ExecutorID,
+		DetailInfo:      &tipb.ExecutorExecutionSummary_ColumnarScanContext{ColumnarScanContext: &columnarScanContext},
+	}
+}
+
 func TestCopRuntimeStats(t *testing.T) {
 	stats := NewRuntimeStatsColl(nil)
 	tableScanID := 1
@@ -719,6 +751,46 @@ func TestVectorSearchStats(t *testing.T) {
 	require.Equal(t, "tiflash_task:{time:0s, loops:0, threads:0}, vector_idx:{load:{total:0ms,from_s3:1,from_disk:0,from_cache:0},search:{total:0ms,visited_nodes:0,discarded_nodes:0},read:{vec_total:0ms,others_total:0ms}}, tiflash_scan:{mvcc_input_rows:0, mvcc_input_bytes:0, mvcc_output_rows:0, local_regions:0, remote_regions:0, tot_learner_read:0ms, region_balance:none, delta_rows:0, delta_bytes:0, segments:0, stale_read_regions:0, tot_build_snapshot:0ms, tot_build_bitmap:0ms, tot_build_inputstream:0ms, min_local_stream:0ms, max_local_stream:0ms, dtfile:{data_scanned_rows:0, data_skipped_rows:0, mvcc_scanned_rows:0, mvcc_skipped_rows:0, lm_filter_scanned_rows:0, lm_filter_skipped_rows:0, tot_rs_index_check:0ms, tot_read:0ms}}", s.String())
 }
 
+func TestColumnarScanContextStats(t *testing.T) {
+	stats := NewRuntimeStatsColl(nil)
+	execSummary := mockExecutorExecutionSummaryForTiFlashColumnar(
+		1, 10, 2, 1,
+		2, 4, 3, 5, 2048,
+		100, 4096, 80,
+		7, 8, 9, 10,
+		11, 12, 13, 14,
+		15, 16, 17,
+		"tablescan_1",
+	)
+	stats.RecordOneCopTask(1, kv.TiFlash, execSummary)
+	stats.RecordOneCopTask(1, kv.TiFlash, mockExecutorExecutionSummaryForTiFlashColumnar(
+		2, 20, 3, 2,
+		4, 6, 2, 4, 1024,
+		10, 2048, 8,
+		1, 2, 3, 4,
+		5, 6, 7, 8,
+		9, 10, 11,
+		"tablescan_1",
+	))
+	s := stats.GetCopStats(1)
+	require.Equal(t, "tiflash_task:{proc max:2ns, min:1ns, avg: 1ns, p80:2ns, p95:2ns, iters:5, tasks:2, threads:3}, columnar_scan:{mvcc_input_rows:110, mvcc_input_bytes:6144, mvcc_output_rows:88, regions:6, read_tasks:10, physical_tables:3, columns:5, user_read_bytes:3072, read_block:8ms, serialize_block:10ms, init_reader:12ms, prefetch:14ms, deserialize_block:28ms, rough_check:{total:16, selected:18, skipped:20, unknown:22}, remote_segments:24, total_segments:26}", s.String())
+
+	zeroStats := NewRuntimeStatsColl(nil)
+	zeroExecSummary := mockExecutorExecutionSummaryForTiFlashColumnar(
+		1, 0, 1, 1,
+		0, 0, 0, 0, 0,
+		0, 0, 0,
+		0, 0, 0, 0,
+		0, 0, 0, 0,
+		0, 0, 0,
+		"tablescan_1",
+	)
+	zeroStats.RecordOneCopTask(1, kv.TiFlash, zeroExecSummary)
+	zeroString := zeroStats.GetCopStats(1).String()
+	require.Contains(t, zeroString, "columnar_scan:{")
+	require.NotContains(t, zeroString, "tiflash_scan:{")
+}
+
 func TestRuntimeStatsWithCommit(t *testing.T) {
 	commitDetail := &util.CommitDetails{
 		GetCommitTsTime: time.Second,

pkg/util/execdetails/runtime_stats.go

@@ -123,9 +123,10 @@ func (e *basicCopRuntimeStats) Clone() RuntimeStats {
 	}
 	if e.tiflashStats != nil {
 		stats.tiflashStats = &TiflashStats{
-			scanContext:    e.tiflashStats.scanContext.Clone(),
-			waitSummary:    e.tiflashStats.waitSummary.Clone(),
-			networkSummary: e.tiflashStats.networkSummary.Clone(),
+			scanContext:         e.tiflashStats.scanContext.Clone(),
+			columnarScanContext: e.tiflashStats.columnarScanContext.Clone(),
+			waitSummary:         e.tiflashStats.waitSummary.Clone(),
+			networkSummary:      e.tiflashStats.networkSummary.Clone(),
 		}
 	}
 	return stats
@@ -148,6 +149,7 @@ func (e *basicCopRuntimeStats) Merge(rs RuntimeStats) {
 			e.tiflashStats = &TiflashStats{}
 		}
 		e.tiflashStats.scanContext.Merge(tmp.tiflashStats.scanContext)
+		e.tiflashStats.columnarScanContext.Merge(tmp.tiflashStats.columnarScanContext)
 		e.tiflashStats.waitSummary.Merge(tmp.tiflashStats.waitSummary)
 		e.tiflashStats.networkSummary.Merge(tmp.tiflashStats.networkSummary)
 	}
@@ -165,6 +167,12 @@ func (e *basicCopRuntimeStats) mergeExecSummary(summary *tipb.ExecutorExecutionS
 		}
 		e.tiflashStats.scanContext.mergeExecSummary(tiflashScanContext)
 	}
+	if columnarScanContext := summary.GetColumnarScanContext(); columnarScanContext != nil {
+		if e.tiflashStats == nil {
+			e.tiflashStats = &TiflashStats{}
+		}
+		e.tiflashStats.columnarScanContext.mergeExecSummary(columnarScanContext)
+	}
 	if tiflashWaitSummary := summary.GetTiflashWaitSummary(); tiflashWaitSummary != nil {
 		if e.tiflashStats == nil {
 			e.tiflashStats = &TiflashStats{}
@@ -246,7 +254,10 @@ func (crs *CopRuntimeStats) String() string {
 						buf.WriteString(", ")
 						buf.WriteString(crs.stats.tiflashStats.networkSummary.String())
 					}
-					if !crs.stats.tiflashStats.scanContext.Empty() {
+					if !crs.stats.tiflashStats.columnarScanContext.Empty() {
+						buf.WriteString(", ")
+						buf.WriteString(crs.stats.tiflashStats.columnarScanContext.String())
+					} else if !crs.stats.tiflashStats.scanContext.Empty() {
 						buf.WriteString(", ")
 						buf.WriteString(crs.stats.tiflashStats.scanContext.String())
 					}

pkg/util/execdetails/tiflash_stats.go

@@ -31,9 +31,34 @@ import (
 
 // TiflashStats contains tiflash execution stats.
 type TiflashStats struct {
-	scanContext    TiFlashScanContext
-	waitSummary    TiFlashWaitSummary
-	networkSummary TiFlashNetworkTrafficSummary
+	scanContext         TiFlashScanContext
+	columnarScanContext TiFlashColumnarScanContext
+	waitSummary         TiFlashWaitSummary
+	networkSummary      TiFlashNetworkTrafficSummary
+}
+
+// TiFlashColumnarScanContext is used to express the table scan information in tiflash columnar read path.
+type TiFlashColumnarScanContext struct {
+	hasStats                bool
+	regions                 uint64
+	readTasks               uint64
+	physicalTables          uint64
+	columns                 uint64
+	userReadBytes           uint64
+	mvccInputRows           uint64
+	mvccInputBytes          uint64
+	mvccOutputRows          uint64
+	totalReadBlockMs        uint64
+	totalSerializeBlockMs   uint64
+	totalInitReaderMs       uint64
+	totalPrefetchMs         uint64
+	roughCheckTotalPacks    uint64
+	roughCheckSelectedPacks uint64
+	roughCheckSkippedPacks  uint64
+	roughCheckUnknownPacks  uint64
+	remoteSegments          uint64
+	totalSegments           uint64
+	totalDeserializeBlockMs uint64
 }
 
 // TiFlashScanContext is used to express the table scan information in tiflash
@@ -534,6 +559,153 @@ func (context *TiFlashScanContext) Empty() bool {
 	return res
 }
 
+// Clone implements the deep copy of * TiFlashColumnarScanContext
+func (context *TiFlashColumnarScanContext) Clone() TiFlashColumnarScanContext {
+	return TiFlashColumnarScanContext{
+		hasStats:                context.hasStats,
+		regions:                 context.regions,
+		readTasks:               context.readTasks,
+		physicalTables:          context.physicalTables,
+		columns:                 context.columns,
+		userReadBytes:           context.userReadBytes,
+		mvccInputRows:           context.mvccInputRows,
+		mvccInputBytes:          context.mvccInputBytes,
+		mvccOutputRows:          context.mvccOutputRows,
+		totalReadBlockMs:        context.totalReadBlockMs,
+		totalSerializeBlockMs:   context.totalSerializeBlockMs,
+		totalInitReaderMs:       context.totalInitReaderMs,
+		totalPrefetchMs:         context.totalPrefetchMs,
+		roughCheckTotalPacks:    context.roughCheckTotalPacks,
+		roughCheckSelectedPacks: context.roughCheckSelectedPacks,
+		roughCheckSkippedPacks:  context.roughCheckSkippedPacks,
+		roughCheckUnknownPacks:  context.roughCheckUnknownPacks,
+		remoteSegments:          context.remoteSegments,
+		totalSegments:           context.totalSegments,
+		totalDeserializeBlockMs: context.totalDeserializeBlockMs,
+	}
+}
+
+func (context *TiFlashColumnarScanContext) String() string {
+	return fmt.Sprintf("columnar_scan:{"+
+		"mvcc_input_rows:%d, "+
+		"mvcc_input_bytes:%d, "+
+		"mvcc_output_rows:%d, "+
+		"regions:%d, "+
+		"read_tasks:%d, "+
+		"physical_tables:%d, "+
+		"columns:%d, "+
+		"user_read_bytes:%d, "+
+		"read_block:%dms, "+
+		"serialize_block:%dms, "+
+		"init_reader:%dms, "+
+		"prefetch:%dms, "+
+		"deserialize_block:%dms, "+
+		"rough_check:{total:%d, selected:%d, skipped:%d, unknown:%d}, "+
+		"remote_segments:%d, "+
+		"total_segments:%d}",
+		context.mvccInputRows,
+		context.mvccInputBytes,
+		context.mvccOutputRows,
+		context.regions,
+		context.readTasks,
+		context.physicalTables,
+		context.columns,
+		context.userReadBytes,
+		context.totalReadBlockMs,
+		context.totalSerializeBlockMs,
+		context.totalInitReaderMs,
+		context.totalPrefetchMs,
+		context.totalDeserializeBlockMs,
+		context.roughCheckTotalPacks,
+		context.roughCheckSelectedPacks,
+		context.roughCheckSkippedPacks,
+		context.roughCheckUnknownPacks,
+		context.remoteSegments,
+		context.totalSegments)
+}
+
+// Merge make sum to merge the information in TiFlashColumnarScanContext
+func (context *TiFlashColumnarScanContext) Merge(other TiFlashColumnarScanContext) {
+	context.hasStats = context.hasStats || other.hasStats
+	context.regions += other.regions
+	context.readTasks += other.readTasks
+	if other.physicalTables > context.physicalTables {
+		context.physicalTables = other.physicalTables
+	}
+	if other.columns > context.columns {
+		context.columns = other.columns
+	}
+	context.userReadBytes += other.userReadBytes
+	context.mvccInputRows += other.mvccInputRows
+	context.mvccInputBytes += other.mvccInputBytes
+	context.mvccOutputRows += other.mvccOutputRows
+	context.totalReadBlockMs += other.totalReadBlockMs
+	context.totalSerializeBlockMs += other.totalSerializeBlockMs
+	context.totalInitReaderMs += other.totalInitReaderMs
+	context.totalPrefetchMs += other.totalPrefetchMs
+	context.roughCheckTotalPacks += other.roughCheckTotalPacks
+	context.roughCheckSelectedPacks += other.roughCheckSelectedPacks
+	context.roughCheckSkippedPacks += other.roughCheckSkippedPacks
+	context.roughCheckUnknownPacks += other.roughCheckUnknownPacks
+	context.remoteSegments += other.remoteSegments
+	context.totalSegments += other.totalSegments
+	context.totalDeserializeBlockMs += other.totalDeserializeBlockMs
+}
+
+func (context *TiFlashColumnarScanContext) mergeExecSummary(summary *tipb.ColumnarScanContext) {
+	if summary == nil {
+		return
+	}
+	context.hasStats = true
+	context.regions += summary.GetRegions()
+	context.readTasks += summary.GetReadTasks()
+	if summary.GetPhysicalTables() > context.physicalTables {
+		context.physicalTables = summary.GetPhysicalTables()
+	}
+	if summary.GetColumns() > context.columns {
+		context.columns = summary.GetColumns()
+	}
+	context.userReadBytes += summary.GetUserReadBytes()
+	context.mvccInputRows += summary.GetMvccInputRows()
+	context.mvccInputBytes += summary.GetMvccInputBytes()
+	context.mvccOutputRows += summary.GetMvccOutputRows()
+	context.totalReadBlockMs += summary.GetTotalReadBlockMs()
+	context.totalSerializeBlockMs += summary.GetTotalSerializeBlockMs()
+	context.totalInitReaderMs += summary.GetTotalInitReaderMs()
+	context.totalPrefetchMs += summary.GetTotalPrefetchMs()
+	context.roughCheckTotalPacks += summary.GetRoughCheckTotalPacks()
+	context.roughCheckSelectedPacks += summary.GetRoughCheckSelectedPacks()
+	context.roughCheckSkippedPacks += summary.GetRoughCheckSkippedPacks()
+	context.roughCheckUnknownPacks += summary.GetRoughCheckUnknownPacks()
+	context.remoteSegments += summary.GetRemoteSegments()
+	context.totalSegments += summary.GetTotalSegments()
+	context.totalDeserializeBlockMs += summary.GetTotalDeserializeBlockMs()
+}
+
+// Empty check whether TiFlashColumnarScanContext is empty.
+func (context *TiFlashColumnarScanContext) Empty() bool {
+	return !context.hasStats &&
+		context.regions == 0 &&
+		context.readTasks == 0 &&
+		context.physicalTables == 0 &&
+		context.columns == 0 &&
+		context.userReadBytes == 0 &&
+		context.mvccInputRows == 0 &&
+		context.mvccInputBytes == 0 &&
+		context.mvccOutputRows == 0 &&
+		context.totalReadBlockMs == 0 &&
+		context.totalSerializeBlockMs == 0 &&
+		context.totalInitReaderMs == 0 &&
+		context.totalPrefetchMs == 0 &&
+		context.roughCheckTotalPacks == 0 &&
+		context.roughCheckSelectedPacks == 0 &&
+		context.roughCheckSkippedPacks == 0 &&
+		context.roughCheckUnknownPacks == 0 &&
+		context.remoteSegments == 0 &&
+		context.totalSegments == 0 &&
+		context.totalDeserializeBlockMs == 0
+}
+
 // TiFlashWaitSummary is used to express all kinds of wait information in tiflash
 type TiFlashWaitSummary struct {
 	// keep execution time to do merge work, always record the wait time with largest execution time