Skip to content

Commit 952d5fa

Browse files
committed
Merge Geant4 scoring meshes from parallel o2-sim workers
This makes Geant4 command-line scoring usable with o2-sim running several workers. - Each worker writes its scoring meshes to <mesh>.worker<pid>.txt before finishing the Geant4 run. - o2-sim sums the worker files into <mesh>.txt once all workers have exited. - The tool o2-sim-merge-g4scoring does the same merge for a given directory. - The total column merges exactly; entries and total^2 count Geant4 events, which o2-sim splits into chunks per worker.
1 parent 956c6bd commit 952d5fa

7 files changed

Lines changed: 238 additions & 1 deletion

File tree

Common/SimConfig/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ o2_add_library(SimConfig
2121
src/InteractionDiamondParam.cxx
2222
src/GlobalProcessCutSimParam.cxx
2323
src/FluenceWeightCalculator.cxx
24+
src/G4ScoringMerger.cxx
2425
PUBLIC_LINK_LIBRARIES O2::CommonUtils
2526
O2::DetectorsCommonDataFormats O2::SimulationDataFormat
2627
FairRoot::Base Boost::program_options)
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Copyright 2019-2026 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
#ifndef O2_SIMCONFIG_G4SCORINGMERGER_H
13+
#define O2_SIMCONFIG_G4SCORINGMERGER_H
14+
15+
#include <string>
16+
17+
namespace o2::conf
18+
{
19+
20+
/// Name of the Geant4 scoring dump written by one simulation worker
21+
std::string g4ScoringWorkerFileName(const std::string& meshName, int pid);
22+
23+
/// Sum the per-worker Geant4 scoring dumps <mesh>.worker<pid>.txt in a directory into <mesh>.txt.
24+
/// Returns the number of merged meshes, or -1 if the worker files are inconsistent.
25+
int mergeG4ScoringDumps(const std::string& directory);
26+
27+
} // namespace o2::conf
28+
29+
#endif
Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
// Copyright 2019-2026 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
#include "SimConfig/G4ScoringMerger.h"
13+
#include <fairlogger/Logger.h>
14+
#include <filesystem>
15+
#include <fstream>
16+
#include <iomanip>
17+
#include <map>
18+
#include <regex>
19+
#include <sstream>
20+
#include <vector>
21+
22+
namespace o2::conf
23+
{
24+
25+
namespace
26+
{
27+
// One scorer block of a Geant4 mesh dump: its header lines and the summed rows
28+
struct ScorerBlock {
29+
std::vector<std::string> header;
30+
std::vector<std::string> keys; // "iZ,iPHI,iR" in file order
31+
std::vector<double> sum;
32+
std::vector<double> sum2;
33+
std::vector<long> entries;
34+
};
35+
36+
// Read one mesh dump into scorer blocks; returns false on a format error
37+
bool readDump(const std::string& fileName, std::vector<std::string>& meshHeader, std::vector<ScorerBlock>& blocks)
38+
{
39+
std::ifstream in(fileName);
40+
if (!in) {
41+
return false;
42+
}
43+
std::string line;
44+
ScorerBlock* current = nullptr;
45+
while (std::getline(in, line)) {
46+
if (line.rfind("# mesh name", 0) == 0) {
47+
meshHeader.push_back(line);
48+
} else if (line.rfind("# primitive scorer name", 0) == 0) {
49+
blocks.emplace_back();
50+
current = &blocks.back();
51+
current->header.push_back(line);
52+
} else if (line.rfind("#", 0) == 0) {
53+
if (!current) {
54+
return false;
55+
}
56+
current->header.push_back(line);
57+
} else if (!line.empty()) {
58+
if (!current) {
59+
return false;
60+
}
61+
// iZ, iPHI, iR, total, total^2, entries
62+
std::vector<std::string> fields;
63+
std::stringstream ss(line);
64+
std::string field;
65+
while (std::getline(ss, field, ',')) {
66+
fields.push_back(field);
67+
}
68+
if (fields.size() != 6) {
69+
return false;
70+
}
71+
current->keys.push_back(fields[0] + "," + fields[1] + "," + fields[2]);
72+
current->sum.push_back(std::stod(fields[3]));
73+
current->sum2.push_back(std::stod(fields[4]));
74+
current->entries.push_back(std::stol(fields[5]));
75+
}
76+
}
77+
return !blocks.empty();
78+
}
79+
} // namespace
80+
81+
std::string g4ScoringWorkerFileName(const std::string& meshName, int pid)
82+
{
83+
return meshName + ".worker" + std::to_string(pid) + ".txt";
84+
}
85+
86+
int mergeG4ScoringDumps(const std::string& directory)
87+
{
88+
namespace fs = std::filesystem;
89+
const std::regex pattern(R"((.+)\.worker([0-9]+)\.txt)");
90+
std::map<std::string, std::vector<fs::path>> filesPerMesh;
91+
for (auto& entry : fs::directory_iterator(directory)) {
92+
std::smatch match;
93+
const auto name = entry.path().filename().string();
94+
if (entry.is_regular_file() && std::regex_match(name, match, pattern)) {
95+
filesPerMesh[match[1]].push_back(entry.path());
96+
}
97+
}
98+
99+
int merged = 0;
100+
for (auto& [mesh, files] : filesPerMesh) {
101+
std::vector<std::string> meshHeader;
102+
std::vector<ScorerBlock> total;
103+
for (auto& file : files) {
104+
std::vector<std::string> header;
105+
std::vector<ScorerBlock> blocks;
106+
if (!readDump(file.string(), header, blocks)) {
107+
LOG(error) << "Cannot read Geant4 scoring dump " << file;
108+
return -1;
109+
}
110+
if (total.empty()) {
111+
meshHeader = header;
112+
total = std::move(blocks);
113+
continue;
114+
}
115+
if (blocks.size() != total.size()) {
116+
LOG(error) << "Geant4 scoring dump " << file << " has a different set of scorers";
117+
return -1;
118+
}
119+
for (size_t b = 0; b < blocks.size(); ++b) {
120+
if (blocks[b].header != total[b].header || blocks[b].keys != total[b].keys) {
121+
LOG(error) << "Geant4 scoring dump " << file << " does not match the mesh layout of the other workers";
122+
return -1;
123+
}
124+
for (size_t i = 0; i < blocks[b].keys.size(); ++i) {
125+
total[b].sum[i] += blocks[b].sum[i];
126+
total[b].sum2[i] += blocks[b].sum2[i];
127+
total[b].entries[i] += blocks[b].entries[i];
128+
}
129+
}
130+
}
131+
132+
const auto outName = (fs::path(directory) / (mesh + ".txt")).string();
133+
std::ofstream out(outName);
134+
out << std::setprecision(16);
135+
for (auto& line : meshHeader) {
136+
out << line << "\n";
137+
}
138+
for (auto& block : total) {
139+
for (auto& line : block.header) {
140+
out << line << "\n";
141+
}
142+
for (size_t i = 0; i < block.keys.size(); ++i) {
143+
out << block.keys[i] << "," << block.sum[i] << "," << block.sum2[i] << "," << block.entries[i] << "\n";
144+
}
145+
}
146+
LOG(info) << "Merged " << files.size() << " Geant4 scoring dumps into " << outName;
147+
++merged;
148+
}
149+
return merged;
150+
}
151+
152+
} // namespace o2::conf

Detectors/gconfig/g4Config.C

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,10 @@ R__LOAD_LIBRARY(libgeant4vmc)
6161
#include "TG4RunConfiguration.h"
6262
#include "SimConfig/G4Params.h"
6363
#include "SimConfig/FluenceWeightCalculator.h"
64+
#include "SimConfig/G4ScoringMerger.h"
65+
#include "G4ScoringManager.hh"
66+
#include "G4VScoringMesh.hh"
67+
#include <unistd.h>
6468
#include "FastSim/G4FastSimulation.h"
6569
#endif
6670
#include "commonConfig.C"
@@ -159,16 +163,30 @@ void Config()
159163
std::cout << "g4Config.C finished" << std::endl;
160164
}
161165

166+
// Write each Geant4 scoring mesh to a file named after this process, so that parallel workers do not overwrite each other
167+
void dumpScoringMeshesPerWorker()
168+
{
169+
auto scoringManager = G4ScoringManager::GetScoringManagerIfExist();
170+
if (!scoringManager) {
171+
return;
172+
}
173+
for (size_t i = 0; i < scoringManager->GetNumberOfMesh(); ++i) {
174+
const auto meshName = scoringManager->GetMesh(i)->GetWorldName();
175+
scoringManager->DumpAllQuantitiesToFile(meshName, o2::conf::g4ScoringWorkerFileName(meshName, getpid()));
176+
}
177+
}
178+
162179
void Terminate()
163180
{
164181
static bool terminated = false;
165182
if (!terminated) {
183+
terminated = true;
166184
std::cout << "Executing G4 terminate\n";
167185
TGeant4* geant4 = dynamic_cast<TGeant4*>(TVirtualMC::GetMC());
168186
if (geant4) {
187+
dumpScoringMeshesPerWorker();
169188
// we need to call finish run for Geant4 ... Since we use ProcessEvent() interface;
170189
geant4->FinishRun();
171190
}
172-
terminated = true;
173191
}
174192
}

run/CMakeLists.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,11 @@ o2_add_executable(serial
6262
COMPONENT_NAME sim
6363
SOURCES o2sim.cxx
6464
PUBLIC_LINK_LIBRARIES internal::allsim)
65+
o2_add_executable(merge-g4scoring
66+
COMPONENT_NAME sim
67+
SOURCES o2sim_mergeg4scoring.cxx
68+
PUBLIC_LINK_LIBRARIES O2::SimConfig)
69+
6570
o2_add_executable(evalmat
6671
COMPONENT_NAME sim
6772
SOURCES o2sim_evalmat.cxx

run/o2sim_mergeg4scoring.cxx

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// Copyright 2019-2026 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
// Sum the per-worker Geant4 scoring dumps of an o2-sim run into one file per mesh
13+
14+
#include "SimConfig/G4ScoringMerger.h"
15+
#include <iostream>
16+
17+
int main(int argc, char* argv[])
18+
{
19+
const std::string directory = argc > 1 ? argv[1] : ".";
20+
const int merged = o2::conf::mergeG4ScoringDumps(directory);
21+
if (merged < 0) {
22+
return 1;
23+
}
24+
std::cout << "merged " << merged << " scoring mesh(es) in " << directory << "\n";
25+
return 0;
26+
}

run/o2sim_parallel.cxx

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
/// @author Sandro Wenzel
1313

14+
#include "SimConfig/G4ScoringMerger.h"
1415
#include <fairmq/TransportFactory.h>
1516
#include <fairmq/Channel.h>
1617
#include <fairmq/Message.h>
@@ -806,6 +807,11 @@ int main(int argc, char* argv[])
806807

807808
LOG(debug) << "ShmManager operation " << o2::utils::ShmManager::Instance().isOperational() << "\n";
808809

810+
// sum the Geant4 scoring meshes written by the individual workers
811+
if (!errored && o2::conf::mergeG4ScoringDumps(".") < 0) {
812+
errored = true;
813+
}
814+
809815
// do a quick check to see if simulation produced something reasonable
810816
// (mainly useful for continuous integration / automated testing suite)
811817
auto returncode = errored ? 1 : checkresult();

0 commit comments

Comments
 (0)