Skip to content

parse perf statistics from scip .out files #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
svigerske merged 7 commits into from
Jul 30, 2026
Merged

parse perf statistics from scip .out files #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
845a9fd
add parsing of normalized time from @10 in scip .out
svigerske May 28, 2026
52faef1
rename NormalizedTimeReader to PerfDataReader to prepare for more par…
svigerske Jul 27, 2026
0eb67fe
parse also MeanFreq and NormalizedFreq
svigerske Jul 27, 2026
685cec0
add NormTime column to evaluation.xml
svigerske Jul 27, 2026
0db84ce
add parsing of raw performance data
svigerske Jul 27, 2026
3ec797a
add another evaluation.xml example that aggregates perf stats
svigerske Jul 27, 2026
3a0d4eb
avoid _ and . in keys
svigerske Jul 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions ipet/Key.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,14 @@
GitHash = "GitHash"
LogFileName = "LogFileName"
LPSolver = "LPSolver"
MeanFreq = "MeanFreq"
MetaData = "MetaData"
MaximumDepth = "MaxDepth"
Mode = "mode"
NodeName = "NodeName"
Nodes = "Nodes"
NormalizedFreq = "NormalizedFreq"
NormalizedTime = "NormalizedTime"
ObjectiveLimit = "Objlimit"
ObjectiveSense = "Objsense"
OptimalValue = "OptVal"
Expand Down
5 changes: 4 additions & 1 deletion ipet/parsing/ReaderManager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
import xml.etree.ElementTree as ElementTree
from .StatisticReader import ErrorFileReader, GapReader, TimeLimitReader, ListReader, \
BestSolInfeasibleReader, MaxDepthReader, MetaDataReader, NodeNameReader, NodesReader, RootNodeFixingsReader, \
SettingsFileReader, TimeToFirstReader, TimeToBestReader, ObjsenseReader, DateTimeReader, SolCheckerReader
SettingsFileReader, TimeToFirstReader, TimeToBestReader, ObjsenseReader, DateTimeReader, SolCheckerReader, \
PerfDataReader, RawPerfDataReader
from .StatisticReader_TableReader import TableReader, CustomTableReader
from .StatisticReader_VariableReader import VariableReader
from .StatisticReader_CustomReader import CustomReader
Expand Down Expand Up @@ -198,6 +199,8 @@ def registerDefaultReaders(self):
TimeLimitReader(),
TimeToFirstReader(),
TimeToBestReader(),
PerfDataReader(),
RawPerfDataReader(),
TraceFileReader()
])

Expand Down
41 changes: 41 additions & 0 deletions ipet/parsing/StatisticReader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -428,6 +428,47 @@ def extractStatistic(self, line):
except TypeError:
pass

class PerfDataReader(StatisticReader):
"""
reads the processed perf statistics, like normalized solution time
"""
name = 'PerfDataReader'
datatype = float
regular_expr = re.compile(r'^@10 meanfreq=([0-9\.]*) nmsfreq=([0-9\.]*) normtime=([0-9\.]*)$')
keys = [Key.MeanFreq, Key.NormalizedFreq, Key.NormalizedTime]

def extractStatistic(self, line):
m = self.regular_expr.match(line)
if m:
for keyidx in range(len(self.keys)):
try:
key = self.keys[keyidx]
value = float(m.group(keyidx+1))
self.addData(key, value)
except TypeError:
pass

class RawPerfDataReader(StatisticReader):
"""
reads the raw perf statistics, e.g., @06 cpu-cycles:u=13572052343862 cpu-cycles:k=68612735800 instructions:u=25079950435035 instructions:k=54294963792 cache-references:u=610557529359 cache-misses:u=320070506935 branch-instructions:u=3638145957323 branch-misses:u=36667745269 L1-dcache-loads=9147033741310 L1-dcache-load-misses=756723588525 L1-icache-load-misses=730865048 dTLB-loads=9147044559864 dTLB-load-misses=3092490807 iTLB-load-misses=4470770 minor-faults:u=10452738 minor-faults:k=86 major-faults:u=8 major-faults:k=0 context-switches=3217 cpu-migrations=94 cycle_activity.cycles_mem_any=13078074806121 cycle_activity.stalls_mem_any=4772553217683 cycle_activity.stalls_total=5226003573246 task-clock=7223006.36
"""
name = 'RawPerfDataReader'
datatype = int
raw_regular_expr = re.compile(r'^@06 (.*)')

def extractStatistic(self, line):
m = self.raw_regular_expr.match(line)
if m:
for keyval in m.group(1).split() :
keyval = keyval.strip().split('=')
if len(keyval) == 2:
try:
key = 'perf:' + keyval[0].replace('_','-').replace('.','-')
value = int(float(keyval[1]))
self.addData(key, value)
except TypeError:
pass

class ListReader(StatisticReader):
"""
reads a list matching a regular expression
Expand Down
2 changes: 1 addition & 1 deletion ipet/parsing/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,5 @@
from .StatisticReader_TableReader import TableReader, CustomTableReader
from .StatisticReader import StatisticReader, ErrorFileReader, GapReader, TimeLimitReader, \
BestSolInfeasibleReader, MaxDepthReader, MetaDataReader, ObjlimitReader, NodesReader, RootNodeFixingsReader, \
SettingsFileReader, TimeToFirstReader, TimeToBestReader, ListReader, ObjsenseReader, DateTimeReader
SettingsFileReader, TimeToFirstReader, TimeToBestReader, PerfDataReader, RawPerfDataReader, ListReader, ObjsenseReader, DateTimeReader
__all__ = []
62 changes: 62 additions & 0 deletions scripts/evaluation-perf.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
<?xml version="1.0" ?>
<Evaluation index="auto">
<Column origcolname="NormalizedTime" minval="0.5" name="NormTime" comp="quot shift. by 1">
<Aggregation aggregation="shmean" name="shmean" shiftby="1.0"/>
</Column>
<Column origcolname="Status" reduction="getBestStatus"/>
<Column origcolname="perf:cpu-cycles:u" minval="1" name="CyclesU" comp="quot shift. by 1000">
<Aggregation aggregation="shmean" name="shmean" shiftby="1000.0"/>
</Column>
<Column origcolname="perf:cpu-cycles:k" minval="1" name="CyclesK" comp="quot shift. by 1000">
<Aggregation aggregation="shmean" name="shmean" shiftby="1000.0"/>
</Column>
<Column origcolname="perf:instructions:u" minval="1" name="InstrU" comp="quot shift. by 1000">
<Aggregation aggregation="shmean" name="shmean" shiftby="1000.0"/>
</Column>
<Column origcolname="perf:instructions:k" minval="1" name="InstrK" comp="quot shift. by 1000">
<Aggregation aggregation="shmean" name="shmean" shiftby="1000.0"/>
</Column>
<Column origcolname="perf:cache-references:u" minval="1" name="CacheRefU" comp="quot shift. by 1000">
<Aggregation aggregation="shmean" name="shmean" shiftby="1000.0"/>
</Column>
<Column origcolname="perf:cache-misses:u" minval="1" name="CacheMissU" comp="quot shift. by 1000">
<Aggregation aggregation="shmean" name="shmean" shiftby="1000.0"/>
</Column>
<Column origcolname="perf:branch-instructions:u" minval="1" name="BranchInstrU" comp="quot shift. by 1000">
<Aggregation aggregation="shmean" name="shmean" shiftby="1000.0"/>
</Column>
<Column origcolname="perf:branch-misses:u" minval="1" name="BranchMissU" comp="quot shift. by 1000">
<Aggregation aggregation="shmean" name="shmean" shiftby="1000.0"/>
</Column>
<Column origcolname="perf:minor-faults:u" minval="0" name="MinorFaultU">
<Aggregation aggregation="mean" name="mean"/>
</Column>
<Column origcolname="perf:minor-faults:k" minval="0" name="MinorFaultK">
<Aggregation aggregation="mean" name="mean"/>
</Column>
<Column origcolname="perf:major-faults:u" minval="0" name="MajorFaultU">
<Aggregation aggregation="mean" name="mean"/>
</Column>
<Column origcolname="perf:major-faults:k" minval="0" name="MajorFaultK">
<Aggregation aggregation="mean" name="mean"/>
</Column>
<Column origcolname="perf:context-switches" minval="0" name="ContextSwitch">
<Aggregation aggregation="mean" name="mean"/>
</Column>
<Column origcolname="perf:cycle-activity-cycles-mem-any" minval="1" name="CycleMemAny" comp="quot shift. by 1000">
<Aggregation aggregation="shmean" name="shmean" shiftby="1000.0"/>
</Column>
<Column origcolname="perf:cycle-activity-stalls-mem-any" minval="1" name="StallMemAny" comp="quot shift. by 1000">
<Aggregation aggregation="shmean" name="shmean" shiftby="1000.0"/>
</Column>
<Column origcolname="perf:cycle-activity-stalls-total" minval="1" name="StallTotal" comp="quot shift. by 1000">
<Aggregation aggregation="shmean" name="shmean" shiftby="1000.0"/>
</Column>
<Column origcolname="perf:task-clock" minval="1" name="TaskClock">
<Aggregation aggregation="mean" name="mean"/>
</Column>
<FilterGroup name="all"/>
<FilterGroup name="alloptimal">
<Filter anytestrun="all" expression1="_solved_" expression2="1" operator="eq"/>
</FilterGroup>
</Evaluation>
3 changes: 3 additions & 0 deletions scripts/evaluation.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
<Column origcolname="SolvingTime" minval="0.5" name="Time" comp="quot shift. by 1">
<Aggregation aggregation="shmean" name="shmean" shiftby="1.0"/>
</Column>
<Column origcolname="NormalizedTime" minval="0.5" name="NormTime" comp="quot shift. by 1">
<Aggregation aggregation="shmean" name="shmean" shiftby="1.0"/>
</Column>
<Column name="Nodes" origcolname="Nodes" comp="quot">
<Aggregation aggregation="shmean" name="shmean" shiftby="100.0" />
</Column>
Expand Down
Loading