REST-for-Physics  v2.3
Rare Event Searches ToolKit for Physics
Loading...
Searching...
No Matches
TRestProcessRunner.cxx
1
24
25#include <TGeoManager.h>
26
27#include "Math/MinimizerOptions.h"
28#include "TBranchRef.h"
29#include "TInterpreter.h"
30#include "TMinuitMinimizer.h"
31#include "TMutex.h"
32#include "TROOT.h"
33#include "TRestManager.h"
34#include "TRestThread.h"
35#include "TRestTools.h"
36
37#ifdef WIN32
38#include <io.h>
39#else
40#include "unistd.h"
41#endif // !WIN32
42
43std::mutex mutex_write;
44
45using namespace std;
46#ifdef TIME_MEASUREMENT
47#include <chrono>
48using namespace std::chrono;
49int deltaTime;
50int writeTime;
51int readTime;
52high_resolution_clock::time_point tS, tE;
53#endif
54
55int ncalculated = 10;
56Long64_t bytesReadLast = 0;
57Double_t progressLast = 0;
58Int_t progressLastPrinted = 0;
59vector<Long64_t> bytesAdded(ncalculated, 0);
60vector<Double_t> progAdded(ncalculated, 0);
61int positionCalculated = 0;
62int printInterval = 200000; // 0.2s
63int inputTreeEntries = 0;
64
65ClassImp(TRestProcessRunner);
66
67TRestProcessRunner::TRestProcessRunner() { Initialize(); }
68
69TRestProcessRunner::~TRestProcessRunner() {}
70
75 fRunInfo = nullptr;
76 fInputEvent = nullptr;
77 fOutputEvent = nullptr;
78 fEventTree = nullptr;
79 fAnalysisTree = nullptr;
80 fOutputDataFile = nullptr;
82
83 fThreads.clear();
84 fProcessInfo.clear();
85
86 fThreadNumber = 0;
87 fFirstEntry = 0;
88 fNFilesSplit = 0;
89 fEventsToProcess = 0;
90 fProcessedEvents = 0;
91 fProcessNumber = 0;
92 fProcStatus = kNormal;
93 fFileSplitSize = 10000000000LL; // 10GB maximum
94 fFileCompression = 2; // default compression level
95
96 fUseTestRun = true;
97 fUsePauseMenu = true;
98 fValidateObservables = false;
99 fSortOutputEvents = true;
100 fInputAnalysisStorage = true;
101 fInputEventStorage = true;
102 fOutputEventStorage = true;
103 fOutputAnalysisStorage = true;
104}
105
116 RESTInfo << RESTendl;
117 if (fHostmgr != nullptr) {
118 fRunInfo = fHostmgr->GetRunInfo();
119 if (fRunInfo == nullptr) {
120 RESTError << "File IO has not been specified, " << RESTendl;
121 RESTError << "please make sure the \"TRestFiles\" section is ahead of the "
122 "\"TRestProcessRunner\" section"
123 << RESTendl;
124 exit(0);
125 }
126 } else {
127 RESTError << "manager not initialized!" << RESTendl;
128 exit(0);
129 }
130
131 // Exiting as soon as possible in order to avoid seg.fault at TRestThread.
132 // And sometimes gets stuck - probably a thread not closed? - and the execution
133 // of restManager does not end even if TRestThread calls exit(1)
134 // I believe it is risky to exit() at TRestThread without closing threads.
135 // It is a guess (J.G.)
136 if (!fRunInfo->GetFileProcess() && fRunInfo->GetEntries() == 0) {
137 RESTError << "TRestProcessRunner::BeginOfInit. The input file is a valid REST file but entries are 0!"
138 << RESTendl;
139 exit(1);
140 }
141
143
144 // firstEntry = StringToInteger(GetParameter("firstEntry", "0"));
145 // eventsToProcess = StringToInteger(GetParameter("eventsToProcess", "0"));
146 int lastEntry = StringToInteger(GetParameter("lastEntry", "0"));
147 if (lastEntry - fFirstEntry > 0 && fEventsToProcess == 0) {
148 fEventsToProcess = lastEntry - fFirstEntry;
149 } else if (fEventsToProcess > 0 && lastEntry - fFirstEntry > 0 &&
150 lastEntry - fFirstEntry != fEventsToProcess) {
151 RESTWarning << "Conflict number of events to process!" << RESTendl;
152 } else if (fEventsToProcess > 0 && lastEntry - fFirstEntry == 0) {
153 lastEntry = fFirstEntry + fEventsToProcess;
154 } else if (fEventsToProcess == 0 && fFirstEntry == 0 && lastEntry == 0) {
155 fEventsToProcess = REST_MAXIMUM_EVENTS;
156 lastEntry = REST_MAXIMUM_EVENTS;
157 } else {
158 RESTWarning << "error setting of event number" << RESTendl;
159 fEventsToProcess = fEventsToProcess > 0 ? fEventsToProcess : REST_MAXIMUM_EVENTS;
160 fFirstEntry = fFirstEntry > 0 ? fFirstEntry : 0;
161 lastEntry = lastEntry == fFirstEntry + fEventsToProcess ? lastEntry : REST_MAXIMUM_EVENTS;
162 }
163 fRunInfo->SetCurrentEntry(fFirstEntry);
164
165 if (fFileSplitSize < 50000000LL || fFileSplitSize > 100000000000LL) {
166 RESTWarning << "automatic file splitting size cannot < 10MB or > 100GB, setting to default (10GB)."
167 << RESTendl;
168 fFileSplitSize = 10000000000LL;
169 }
170
171 // fUseTestRun = StringToBool(GetParameter("useTestRun", "ON"));
172 // fUsePauseMenu = StringToBool(GetParameter("usePauseMenu", "OFF"));
174 fProcStatus = kIgnore;
175 if (!fOutputAnalysisStorage) {
176 RESTError << "output analysis must be turned on to process data!" << RESTendl;
177 exit(1);
178 }
179 // fValidateObservables = StringToBool(GetParameter("validateObservables", "OFF"));
180 // fSortOutputEvents = StringToBool(GetParameter("sortOutputEvents", "ON"));
181 // fThreadNumber = StringToDouble(GetParameter("threadNumber", "1"));
182 // if (ToUpper(GetParameter("inputAnalysis", "ON")) == "ON") fOutputItem[0] = true;
183 // if (ToUpper(GetParameter("inputEvent", "OFF")) == "ON") fOutputItem[1] = true;
184 // if (ToUpper(GetParameter("outputEvent", "ON")) == "ON") fOutputItem[2] = true;
185 // fOutputItem[3] = true;
186
187 // fOutputItem = Split(GetParameter("treeBranches",
188 // "inputevent:outputevent:inputanalysis"), ":");
189 if (fThreadNumber < 1) {
190 fThreadNumber = 1;
191 }
192 if (fThreadNumber > 15) {
193 fThreadNumber = 15;
194 }
195
196 for (int i = 0; i < fThreadNumber; i++) {
197 TRestThread* t = new TRestThread();
198 t->SetProcessRunner(this);
199 t->SetVerboseLevel(fVerboseLevel);
200 t->SetThreadId(i);
201 t->SetCompressionLevel(fFileCompression);
202 fThreads.push_back(t);
203 }
204}
205
214Int_t TRestProcessRunner::ReadConfig(const string& keydeclare, TiXmlElement* e) {
215 if (keydeclare == "addProcess") {
216 string active = GetParameter("value", e, "");
217 if (active != "" && ToUpper(active) != "ON") return 0;
218
219 string processName = GetParameter("name", e, "");
220
221 string processType = GetParameter("type", e, "");
222
223 if (processType == "") {
224 RESTWarning << "Bad expression of addProcess" << RESTendl;
225 return 0;
226 } else if (processName == "") {
227 RESTWarning << "Event process " << processType << " has no name, it will be skipped" << RESTendl;
228 return 0;
229 }
230
231 RESTInfo << "adding process " << processType << " \"" << processName << "\"" << RESTendl;
232 vector<TRestEventProcess*> processes;
233 for (int i = 0; i < fThreadNumber; i++) {
234 TRestEventProcess* p = InstantiateProcess(processType, e);
235 if (p != nullptr) {
236 if (p->isExternal()) {
237 fRunInfo->SetExtProcess(p);
238 return 0;
239 }
241 p->singleThreadOnly())) {
242 fUsePauseMenu = false;
243 fProcStatus = kIgnore;
244 if (fThreadNumber > 1) {
245 RESTInfo << "multi-threading is disabled due to process \"" << p->GetName() << "\""
246 << RESTendl;
247 RESTInfo << "This process is in debug mode or is single thread only" << RESTendl;
248 for (i = fThreadNumber; i > 1; i--) {
249 // delete (*fThreads.end());
250 fThreads.erase(fThreads.end() - 1);
251 fThreadNumber--;
252 }
253 }
254 }
255 processes.push_back(p);
256 } else {
257 return 1;
258 }
259 }
260
261 for (int i = 0; i < fThreadNumber; i++) {
262 TRestEventProcess* p = processes[i];
263 for (int j = 0; j < fThreadNumber; j++) {
264 p->SetParallelProcess(processes[j]);
265 }
266 fThreads[i]->AddProcess(p);
267 }
268
269 fProcessNumber++;
270 RESTDebug << "process \"" << processType << "\" has been added!" << RESTendl;
271 return 0;
272 }
273
274 return 0;
275}
276
284 RESTDebug << "Validating process chain..." << RESTendl;
285
286 if (fRunInfo->GetFileProcess() != nullptr) {
287 fInputEvent = fRunInfo->GetFileProcess()->GetOutputEvent();
288 } else {
289 if (fThreads[0]->GetProcessnum() > 0 &&
290 fThreads[0]->GetProcess(0)->GetInputEvent().address != nullptr) {
291 string name = fThreads[0]->GetProcess(0)->GetInputEvent().type;
293 a->Initialize();
294 fRunInfo->SetInputEvent(a);
295 }
296 fInputEvent = fRunInfo->GetInputEvent();
297 }
298 if (fInputEvent == nullptr) {
299 RESTError << "Cannot determine input event, validating process chain failed!" << RESTendl;
300 exit(1);
301 }
302
303 if (fProcessNumber > 0) {
304 if (fThreads[0]->ValidateChain(fInputEvent) == -1) exit(1);
305 }
306
307 ReadProcInfo();
308}
309
316 if (fRunInfo->GetFileProcess() != nullptr) {
317 fProcessInfo["FirstProcess"] = fRunInfo->GetFileProcess()->GetName();
318 } else {
319 if (fProcessNumber > 0) fProcessInfo["FirstProcess"] = fThreads[0]->GetProcess(0)->GetName();
320 }
321 int n = fProcessNumber;
322 fProcessInfo["LastProcess"] =
323 (n == 0 ? fProcessInfo["FirstProcess"] : fThreads[0]->GetProcess(n - 1)->GetName());
324 fProcessInfo["ProcNumber"] = ToString(n);
325}
326
344 RESTDebug << "Creating output File " << fRunInfo->GetOutputFileName() << RESTendl;
345
346 TString filename = fRunInfo->FormFormat(fRunInfo->GetOutputFileName());
347 fOutputDataFileName = filename;
348 auto output = TRestRootFileHandle::Open(filename.Data(), TRestRootFileMode::Recreate);
349 if (!output) {
350 RESTError << output.Error() << RESTendl;
351 exit(1);
352 }
353 if (fOutputDataFileOwner && !fOutputDataFileOwner.Close()) {
354 RESTError << fOutputDataFileOwner.Error() << RESTendl;
355 exit(1);
356 }
357 fOutputDataFileOwner = std::move(output);
358 fOutputDataFile = fOutputDataFileOwner.Get();
359 // set compression level here will cause problem in pipeline
360 // we must set in each threadCompression
361 // fOutputDataFile->SetCompressionLevel(fFile);
362 if (!fOutputDataFile->IsOpen()) {
363 RESTError << "Failed to create output file: " << fOutputDataFile->GetName() << RESTendl;
364 exit(1);
365 }
366 RESTInfo << RESTendl;
367 RESTInfo << "TRestProcessRunner : preparing threads..." << RESTendl;
368 fRunInfo->ResetEntry();
369 fRunInfo->SetCurrentEntry(fFirstEntry);
370 for (int i = 0; i < fThreadNumber; i++) {
371 fThreads[i]->PrepareToProcess(&fInputAnalysisStorage);
372 }
373
374 // print metadata
375 if (fRunInfo->GetFileProcess() != nullptr) {
376 RESTEssential << this->ClassName() << ": 1 + " << fProcessNumber << " processes loaded, "
377 << fThreadNumber << " threads prepared!" << RESTendl;
378 } else {
379 RESTEssential << this->ClassName() << ": " << fProcessNumber << " processes loaded, " << fThreadNumber
380 << " threads prepared!" << RESTendl;
381 }
383 if (fRunInfo->GetFileProcess() != nullptr) fRunInfo->GetFileProcess()->PrintMetadata();
384
385 for (int i = 0; i < fProcessNumber; i++) {
386 fThreads[0]->GetProcess(i)->PrintMetadata();
387 }
388 } else {
389 if (fRunInfo->GetFileProcess() != nullptr) {
390 RESTcout << "(external) " << fRunInfo->GetFileProcess()->ClassName() << " : "
391 << fRunInfo->GetFileProcess()->GetName() << RESTendl;
392 }
393 for (int i = 0; i < fProcessNumber; i++) {
394 RESTcout << "++ " << fThreads[0]->GetProcess(i)->ClassName() << " : "
395 << fThreads[0]->GetProcess(i)->GetName() << RESTendl;
396 }
397 }
398 RESTcout << "=" << RESTendl;
399
400 // copy thread's event tree to local
401 fOutputDataFile->cd();
402 TTree* tree = fThreads[0]->GetEventTree();
403 if (tree != nullptr) {
404 fEventTree = (TTree*)tree->Clone();
405 fEventTree->SetName("EventTree");
406 fEventTree->SetTitle("REST Event Tree");
407 fEventTree->SetDirectory(fOutputDataFile);
408 TTree::SetMaxTreeSize(100000000000LL > fFileSplitSize * 2
409 ? 100000000000LL
410 : fFileSplitSize * 2); // the default size is 100GB
411 } else {
412 fEventTree = nullptr;
413 }
414
415 // initialize analysis tree
416 fAnalysisTree = new TRestAnalysisTree("AnalysisTree", "REST Process Analysis Tree");
417 fAnalysisTree->SetDirectory(fOutputDataFile);
418 TRestAnalysisTree::SetMaxTreeSize(100000000000LL > fFileSplitSize * 2 ? 100000000000LL
419 : fFileSplitSize * 2);
420
421 tree = fThreads[0]->GetAnalysisTree();
422 if (tree != nullptr) {
423 fNBranches = tree->GetNbranches();
424 } else {
425 RESTError << "Threads are not initialized! No AnalysisTree!" << RESTendl;
426 exit(1);
427 }
428
429 fOutputDataFile->cd();
430 if (fEventTree != nullptr) {
431 fEventTree->Write(nullptr, kOverwrite);
432 }
433 if (fAnalysisTree != nullptr) {
434 fAnalysisTree->Write(nullptr, kOverwrite);
435 }
436
437 // reset runner
438 this->ResetRunTimes();
439 fProcessedEvents = 0;
440 fRunInfo->ResetEntry();
441 fRunInfo->SetCurrentEntry(fFirstEntry);
442 inputTreeEntries = fRunInfo->GetEntries();
443
444 // set root mutex
446 ROOT::Math::MinimizerOptions::SetDefaultMinimizer("Minuit");
447 TMinuitMinimizer::UseStaticMinuit(false);
448 if (gGlobalMutex == nullptr) {
449 gGlobalMutex = new TMutex(true);
450 gROOTMutex = gGlobalMutex;
451 gInterpreterMutex = gGlobalMutex;
452 }
453
454#ifdef TIME_MEASUREMENT
455 high_resolution_clock::time_point t3 = high_resolution_clock::now();
456#endif
457
458 // start the thread!
459 RESTcout << this->ClassName() << ": Starting the Process.." << RESTendl;
460 for (int i = 0; i < fThreadNumber; i++) {
461 fThreads[i]->StartThread();
462 }
463
464 while (fProcStatus == kPause ||
465 (fRunInfo->GetInputEvent() != nullptr && fEventsToProcess > fProcessedEvents)) {
467
468 if (fProcStatus == kNormal && Console::kbhit()) // if keyboard inputs
469 {
470 int a = Console::ReadKey(); // get char
471
472 if (a == 'p') {
473 fProcStatus = kPause;
474 usleep(100000); // wait 0.1s for the processes to finish;
476 "| |", TRestStringOutput::REST_Display_Orientation::kMiddle);
477 Console::ClearLinesAfterCursor();
478 RESTLog << RESTendl;
479 RESTLog << "-" << RESTendl;
480 RESTLog << "PROCESS PAUSED!" << RESTendl;
481 RESTLog << "-" << RESTendl;
482 RESTLog << " " << RESTendl;
483 }
484 }
485
486 if (fProcStatus == kPause) {
487 PauseMenu();
488 }
489 if (fProcStatus == kStopping) {
490 break;
491 }
492
493 usleep(printInterval);
494
495 // cout << eventsToProcess << " " << fProcessedEvents << " " << lastEntry <<
496 // " " << fCurrentEvent << endl; cout << fProcessedEvents << "\r";
497 // printf("%d processed events now...\r", fProcessedEvents); fflush(stdout);
498 }
499
500 if (fProcStatus != kIgnore && Console::kbhit())
501 while (getchar() != '\n')
502 ; // clear buffer
503
504 RESTEssential << "Waiting for processes to finish ..." << RESTendl;
505
506 while (true) {
507 usleep(100000);
508 bool finish = fThreads[0]->Finished();
509 for (int i = 1; i < fThreadNumber; i++) {
510 finish = finish && fThreads[i]->Finished();
511 }
512 if (finish) break;
513 }
514
515 // make dummy analysis tree filled with observables
516 fAnalysisTree->GetEntry(fAnalysisTree->GetEntries() - 1);
517 // call EndProcess() for all processes
518 for (int i = 0; i < fThreadNumber; i++) {
519 fThreads[i]->EndProcess();
520 }
521 if (fRunInfo->GetFileProcess()) {
522 fRunInfo->GetFileProcess()->EndProcess();
523 }
524 fProcStatus = kFinished;
525
526#ifdef TIME_MEASUREMENT
527 high_resolution_clock::time_point t4 = high_resolution_clock::now();
528 deltaTime = (int)duration_cast<microseconds>(t4 - t3).count();
529#endif
530
531 // reset the mutex to null
532 delete gGlobalMutex;
533 gGlobalMutex = nullptr;
534 gROOTMutex = nullptr;
535 gInterpreterMutex = nullptr;
536
537 RESTcout << this->ClassName() << ": " << fProcessedEvents << " processed events" << RESTendl;
538
539#ifdef TIME_MEASUREMENT
540 RESTInfo << "Total processing time : " << ((Double_t)deltaTime) / 1000. << " ms" << RESTendl;
541 RESTInfo << "Average read time from disk (per event) : "
542 << ((Double_t)readTime) / fProcessedEvents / 1000. << " ms" << RESTendl;
543 RESTInfo << "Average process time (per event) : "
544 << ((Double_t)(deltaTime - readTime - writeTime)) / fProcessedEvents / 1000. << " ms"
545 << RESTendl;
546 RESTInfo << "Average write time to disk (per event) : "
547 << ((Double_t)writeTime) / fProcessedEvents / 1000. << " ms" << RESTendl;
548 RESTInfo << "=" << RESTendl;
549#endif
550
551 if (fRunInfo->GetOutputFileName() != "/dev/null") {
553 }
554}
555
567 TRestStringOutput::REST_Display_Orientation::kMiddle);
568 Console::ClearLinesAfterCursor();
569
570 RESTLog << "--------------MENU--------------" << RESTendl;
571 RESTLog << "\"v\" : change the verbose level" << RESTendl;
572 RESTLog << "\"n\" : push foward one event, then pause" << RESTendl;
573 RESTLog << "\"l\" : print the latest processed event" << RESTendl;
574 RESTLog << "\"d\" : detach the current process" << RESTendl;
575 RESTLog << "\"q\" : stop and quit the process" << RESTendl;
576 RESTLog << "press \"p\" to continue process..." << RESTendl;
577 RESTLog << "-" << RESTendl;
578 RESTLog << RESTendl;
579 RESTLog << RESTendl;
580 int menuupper = 15;
581 int infobar = 11;
582 while (true) {
583 Console::ClearCurrentLine();
584 int b = Console::ReadKey(); // no need to press enter
585
586 if (b == 'v') {
587 Console::CursorUp(infobar);
588 RESTLog.setcolor(COLOR_BOLDGREEN);
589 RESTLog << "Changing verbose level for all the processes..." << RESTendl;
590 RESTLog.setcolor(COLOR_BOLDWHITE);
591 Console::CursorDown(1);
592 Console::ClearLinesAfterCursor();
593 RESTLog << "type \"0\"/\"s\" to set level silent" << RESTendl;
594 RESTLog << "type \"1\"/\"e\" to set level essential" << RESTendl;
595 RESTLog << "type \"2\"/\"i\" to set level info" << RESTendl;
596 RESTLog << "type \"3\"/\"d\" to set level debug" << RESTendl;
597 RESTLog << "type \"4\"/\"x\" to set level extreme" << RESTendl;
598 RESTLog << "type other to return the pause menu" << RESTendl;
599 RESTLog << "-" << RESTendl;
600 RESTLog << RESTendl;
601 RESTLog << RESTendl;
602 while (true) {
603 Console::CursorUp(1);
604 int c = Console::Read();
605 if (c != '\n')
606 while (Console::Read() != '\n')
607 ;
609 if (c == '0' || c == 's') {
611 } else if (c == '1' || c == 'e') {
613 } else if (c == '2' || c == 'i') {
615 } else if (c == '3' || c == 'd') {
617 } else if (c == '4' || c == 'x') {
619 } else {
620 Console::CursorUp(infobar);
621 RESTLog.setcolor(COLOR_BOLDYELLOW);
622 RESTLog << "Verbose level not set!" << RESTendl;
623 RESTLog.setcolor(COLOR_BOLDWHITE);
624 break;
625 }
626
627 gVerbose = l;
628 this->SetVerboseLevel(l);
629 for (int i = 0; i < fThreadNumber; i++) {
630 fThreads[i]->SetVerboseLevel(l);
631 for (int j = 0; j < fThreads[i]->GetProcessnum(); j++) {
632 fThreads[i]->GetProcess(j)->SetVerboseLevel(l);
633 }
634 }
635 Console::CursorUp(infobar);
636 RESTLog.setcolor(COLOR_BOLDGREEN);
637 RESTLog << "Verbose level has been set to " << ToString(static_cast<int>(l)) << "!"
638 << RESTendl;
639 RESTLog.setcolor(COLOR_BOLDWHITE);
640 break;
641 }
642 Console::ClearLinesAfterCursor();
643 break;
644 } else if (b == 'd') {
645 Console::CursorUp(infobar);
646 RESTLog.setcolor(COLOR_BOLDGREEN);
647 RESTLog << "Detaching restManager to backend" << RESTendl;
648 RESTLog.setcolor(COLOR_BOLDWHITE);
649 Console::CursorDown(1);
650 Console::ClearLinesAfterCursor();
651 RESTLog << "type filename for output redirect" << RESTendl;
652 RESTLog << "leave blank to redirect to /dev/null" << RESTendl;
653 RESTLog << " " << RESTendl;
654 RESTLog << " " << RESTendl;
655 RESTLog << " " << RESTendl;
656 RESTLog << " " << RESTendl;
657 RESTLog << "-" << RESTendl;
658 RESTLog << RESTendl;
659 RESTLog << RESTendl;
660
661 string file;
662
663 Console::CursorUp(1);
664 file = Console::ReadLine();
665 if (file == "") file = "/dev/null";
666 if (TRestTools::fileExists(file)) {
667 if (!TRestTools::isPathWritable(file)) {
668 Console::CursorUp(infobar);
669 RESTLog.setcolor(COLOR_BOLDYELLOW);
670 RESTLog << "file not writeable!" << RESTendl;
671 RESTLog.setcolor(COLOR_BOLDWHITE);
672 break;
673 }
674 } else {
676 Console::CursorUp(infobar);
677 RESTLog.setcolor(COLOR_BOLDYELLOW);
678 RESTLog << "path not writeable!" << RESTendl;
679 RESTLog.setcolor(COLOR_BOLDWHITE);
680 break;
681 }
682 }
683
684#ifdef WIN32
685 RESTWarning << "fork not available on windows!" << RESTendl;
686#else
687 pid_t pid;
688 pid = fork();
689 if (pid < 0) {
690 perror("fork error:");
691 exit(1);
692 }
693 // child process
694 if (pid == 0) {
695 RESTcout << "Child process created! pid: " << getpid() << RESTendl;
696 RESTInfo << "Restarting threads" << RESTendl;
697 mutex_write.unlock();
698 for (int i = 0; i < fThreadNumber; i++) {
699 fThreads[i]->StartThread();
700 }
701 RESTInfo << "Re-directing output to " << file << RESTendl;
702 FILE* f = freopen(file.c_str(), "w", stdout);
703 if (f == nullptr) RESTWarning << "Couldnt redirect output for file: " << file << RESTendl;
704 REST_Display_CompatibilityMode = true;
705 }
706 // father process
707 else {
708 exit(0);
709 }
710 fProcStatus = kNormal;
711 RESTInfo << "Continue processing..." << RESTendl;
712
713#endif // WIN32
714
715 break;
716 } else if (b == 'n') {
717 fProcStatus = kStep;
718 break;
719 } else if (b == 'l') {
720 // Console::ClearScreen();
721 fOutputEvent->PrintEvent();
722 break;
723 } else if (b == 'q') {
724 fProcStatus = kStopping;
725 break;
726 } else if (b == 'p') {
727 Console::CursorUp(menuupper);
728 Console::ClearLinesAfterCursor();
730 fProcStatus = kIgnore;
731 } else {
732 fProcStatus = kNormal;
733 }
734 break;
735 } else if (b == '\n') {
736 // CursorUp(1);
737 } else {
738 Console::CursorUp(infobar);
739 RESTLog.setcolor(COLOR_BOLDYELLOW);
740 RESTLog << "Invailed option \"" << (char)b << "\" (key value: " << b << ") !" << RESTendl;
741 RESTLog.setcolor(COLOR_BOLDWHITE);
742 Console::CursorDown(infobar - 1);
743 }
744 }
745}
746
771 mutex_write.lock(); // lock on
772 while (fProcStatus == kPause) {
773 usleep(100000);
774 }
775#ifdef TIME_MEASUREMENT
776 high_resolution_clock::time_point t1 = high_resolution_clock::now();
777#endif
778 int n;
779 if (fProcessedEvents >= fEventsToProcess || targetevt == nullptr || fProcStatus == kStopping) {
780 n = -1;
781 } else {
782 if (!fInputAnalysisStorage) {
783 n = fRunInfo->GetNextEvent(targetevt, nullptr);
784 } else {
785 n = fRunInfo->GetNextEvent(targetevt, targettree);
786 }
787 }
788
789#ifdef TIME_MEASUREMENT
790 high_resolution_clock::time_point t2 = high_resolution_clock::now();
791 readTime += (int)duration_cast<microseconds>(t2 - t1).count();
792#endif
793 mutex_write.unlock();
794 return n;
795}
796
804 if (fSortOutputEvents) {
805 // Make sure the thread has the minimum event id in the all the
806 // threads. Otherwise, just wait.
807 while (true) {
808 bool smallest = true;
809 for (TRestThread* otherT : fThreads) {
810 if (otherT->Finished()) {
811 continue;
812 }
813 if (t->GetThreadId() == otherT->GetThreadId()) {
814 continue;
815 }
816 int id1 = t->GetInputEvent()->GetID();
817 int id2 = otherT->GetInputEvent()->GetID();
818 if (id1 > id2) {
819 smallest = false;
820 } else if (id1 == id2) {
821 cout << "warning! Two events with same event id! output events maybe dis-ordered!"
822 << endl;
823 }
824 }
825
826 if (smallest) break;
827 usleep(1);
828 }
829 }
830
831 // Start event saving, entering mutex lock region.
832 mutex_write.lock();
833#ifdef TIME_MEASUREMENT
834 high_resolution_clock::time_point t5 = high_resolution_clock::now();
835#endif
836 if (t->GetOutputEvent() != nullptr) {
837 fOutputEvent = t->GetOutputEvent();
838 // copy address of analysis tree of the given thread
839 // to the local tree, then fill the local tree
840 TObjArray* branchesT;
841 TObjArray* branchesL;
842
843 if (fAnalysisTree != nullptr) {
844 TRestAnalysisTree* remotetree = t->GetAnalysisTree();
845
846 // t->GetAnalysisTree()->SetEventInfo(t->GetOutputEvent());
847 // branchesT = t->GetAnalysisTree()->GetListOfBranches();
848 // branchesL = fAnalysisTree->GetListOfBranches();
849 // for (int i = 0; i < nBranches; i++) {
850 // TBranch* branchT = (TBranch*)branchesT->UncheckedAt(i);
851 // TBranch* branchL = (TBranch*)branchesL->UncheckedAt(i);
852 // branchL->SetAddress(branchT->GetAddress());
853 //}
854 fAnalysisTree->SetEventInfo(fOutputEvent);
855 for (int n = 0; n < remotetree->GetNumberOfObservables(); n++) {
856 fAnalysisTree->SetObservable(n, remotetree->GetObservable(n));
857 }
858
859 fAnalysisTree->Fill();
860 }
861
862 if (fEventTree != nullptr) {
863 // t->GetEventTree()->FillEvent(t->GetOutputEvent());
864 branchesT = t->GetEventTree()->GetListOfBranches();
865 branchesL = fEventTree->GetListOfBranches();
866 for (int i = 0; i < branchesT->GetLast() + 1; i++) {
867 TBranch* branchT = (TBranch*)branchesT->UncheckedAt(i);
868 TBranch* branchL = (TBranch*)branchesL->UncheckedAt(i);
869 branchL->SetAddress(branchT->GetAddress());
870 }
871 fEventTree->Fill();
872 }
873 fProcessedEvents++;
874
875 // cout << fTempOutputDataFile << " " << fTempOutputDataFile->GetEND() << " " <<
876 // fAnalysisTree->GetDirectory() << endl; cout << fAutoSplitFileSize << endl; switch file if file size
877 // reaches target
878 if (fOutputDataFile->GetEND() > fFileSplitSize) {
879 if (fAnalysisTree->GetDirectory() == (TDirectory*)fOutputDataFile) {
880 fNFilesSplit++;
881 cout << "TRestProcessRunner: file size reaches limit (" << fFileSplitSize
882 << " bytes), switching to new file with index " << fNFilesSplit << endl;
883
884 // wait 0.1s for the process to finish
885 usleep(100000);
886 for (auto th : fThreads) {
887 for (int j = 0; j < fProcessNumber; j++) {
888 auto proc = th->GetProcess(j);
889 proc->NotifyAnalysisTreeReset();
890 }
891 }
892
893 fAnalysisTree->AutoSave();
894 fAnalysisTree->Reset();
895
896 if (fEventTree != nullptr) {
897 fEventTree->AutoSave();
898 fEventTree->Reset();
899 }
900
901 // write some information to the first(main) data file
902 fRunInfo->SetNFilesSplit(fNFilesSplit);
903 if (fOutputDataFile->GetName() != fOutputDataFileName) {
904 auto mainFile =
905 TRestRootFileHandle::Open(fOutputDataFileName.Data(), TRestRootFileMode::Update);
906 if (!mainFile) {
907 RESTError << mainFile.Error() << RESTendl;
908 exit(1);
909 }
910 WriteProcessesMetadata(mainFile.Get());
911 mainFile->Write(0, TObject::kOverwrite);
912 if (!mainFile.Close()) {
913 RESTError << mainFile.Error() << RESTendl;
914 exit(1);
915 }
916 } else {
918 }
919
920 const std::string splitFileName =
921 std::string(fOutputDataFileName.Data()) + "." + ToString(fNFilesSplit);
922 auto splitFile = TRestRootFileHandle::Open(splitFileName, TRestRootFileMode::Recreate);
923 if (!splitFile) {
924 RESTError << splitFile.Error() << RESTendl;
925 exit(1);
926 }
927 TFile* newfile = splitFile.Get();
928
929 TBranch* branch = nullptr;
930 fAnalysisTree->SetDirectory(newfile);
931 TIter nextb1(fAnalysisTree->GetListOfBranches());
932 while ((branch = (TBranch*)nextb1())) {
933 branch->SetFile(newfile);
934 }
935 if (fAnalysisTree->GetBranchRef()) {
936 fAnalysisTree->GetBranchRef()->SetFile(newfile);
937 }
938
939 if (fEventTree != nullptr) {
940 fEventTree->SetDirectory(newfile);
941 TIter nextb2(fEventTree->GetListOfBranches());
942 while ((branch = (TBranch*)nextb2())) {
943 branch->SetFile(newfile);
944 }
945 if (fEventTree->GetBranchRef()) {
946 fEventTree->GetBranchRef()->SetFile(newfile);
947 }
948 }
949
950 fOutputDataFile->Write(nullptr, TObject::kOverwrite);
951 if (!fOutputDataFileOwner.Close()) {
952 RESTError << fOutputDataFileOwner.Error() << RESTendl;
953 exit(1);
954 }
955 fOutputDataFileOwner = std::move(splitFile);
956 fOutputDataFile = fOutputDataFileOwner.Get();
957 } else {
958 RESTError << "internal error!" << RESTendl;
959 }
960 }
961
962#ifdef TIME_MEASUREMENT
963 high_resolution_clock::time_point t6 = high_resolution_clock::now();
964 writeTime += (int)duration_cast<microseconds>(t6 - t5).count();
965#endif
966
967 if (fProcStatus == kStep) {
968 fProcStatus = kPause;
969 cout << "Processed events:" << fProcessedEvents << endl;
970 }
971 }
972 mutex_write.unlock();
973}
974
981 RESTEssential << "Configuring output file, writing metadata and tree objects" << RESTendl;
982#ifdef TIME_MEASUREMENT
983 fProcessInfo["ProcessTime"] = ToString(deltaTime) + "ms";
984#endif
985 // write the last tree
986 fOutputDataFile->cd();
987 if (fEventTree != nullptr) {
988 fEventTree->Write(nullptr, kOverwrite);
989 }
990 if (fAnalysisTree != nullptr) {
991 fAnalysisTree->Write(nullptr, kOverwrite);
992 }
993
994 // close file
995 fOutputDataFile->Write();
996 if (!fOutputDataFileOwner.Close()) {
997 RESTError << fOutputDataFileOwner.Error() << RESTendl;
998 exit(1);
999 }
1000 fOutputDataFile = nullptr;
1001
1002 // merge process's data file to the main file
1003 // we must call this method before writing process metadata,
1004 // otherwise there would be problems of streamer missing
1005 // https://github.com/rest-for-physics/framework/issues/348
1006 fRunInfo->SetNFilesSplit(fNFilesSplit);
1008
1009 // write metadata
1011
1012 // Write geometry
1013 if (gGeoManager != nullptr) {
1014 gGeoManager->Write("Geometry", TObject::kOverwrite);
1015 }
1016}
1017
1022 TFile* output = destination != nullptr ? destination : fOutputDataFile;
1023 if (output == nullptr || !output->IsOpen() || !output->IsWritable()) {
1024 RESTError << "Cannot write process metadata without a writable destination" << RESTendl;
1025 return;
1026 }
1027 output->cd();
1028
1029 this->Write(nullptr, TObject::kWriteDelete);
1030
1031 if (fRunInfo != nullptr && fRunInfo->GetFileProcess() != nullptr) {
1032 fRunInfo->GetFileProcess()->Write(nullptr, kOverwrite);
1033 }
1034 for (int i = 0; i < fProcessNumber && !fThreads.empty(); i++) {
1035 fThreads[0]->GetProcess(i)->Write(nullptr, kOverwrite);
1036 }
1037}
1038
1044 RESTEssential << "Merging thread files together" << RESTendl;
1045 // add threads file
1046 // processes may have their own TObject output. They are stored in the threads
1047 // file these files are mush smaller that data file, so they are merged to the
1048 // data file.
1049 vector<string> files_to_merge;
1050 for (int i = 0; i < fThreadNumber; i++) {
1051 TFile* f = fThreads[i]->GetOutputFile();
1052 if (f != nullptr) {
1053 const std::string threadFileName = f->GetName();
1054 f->Write(nullptr, TObject::kOverwrite);
1055 if (!fThreads[i]->CloseOutputFile()) {
1056 RESTError << "Failed to close thread output file " << threadFileName << RESTendl;
1057 exit(1);
1058 }
1059 files_to_merge.push_back(threadFileName);
1060 }
1061 }
1062
1064 fOutputDataFile = fRunInfo->MergeToOutputFile(files_to_merge, (string)fOutputDataFileName);
1065 } else {
1066 RESTError << "Output file: " << fOutputDataFileName << " is lost?" << RESTendl;
1067 }
1068}
1069
1070// tools
1075#ifdef TIME_MEASUREMENT
1076 readTime = 0;
1077 writeTime = 0;
1078 deltaTime = 0;
1079#endif
1080 time_t tt = time(nullptr);
1081 fProcessInfo["ProcessDate"] = Split(ToDateTimeString(tt), " ")[0];
1082}
1083
1091 TRestEventProcess* pc = REST_Reflection::Assembly((string)type);
1092 if (pc == nullptr) return nullptr;
1093
1095 pc->SetRunInfo(this->fRunInfo);
1096 pc->SetHostmgr(fHostmgr);
1097 pc->SetObservableValidation(fValidateObservables);
1099
1100 return pc;
1101}
1102
1103double TRestProcessRunner::GetReadingSpeed() {
1104 Long64_t bytes = 0;
1105 for (auto& n : bytesAdded) bytes += n;
1106 double speedbyte = bytes / (double)printInterval * (double)1000000 / ncalculated;
1107 return speedbyte;
1108}
1109
1115 if (fProcStatus == kNormal || fProcStatus == kIgnore) {
1116 // clearLinesAfterCursor();
1117
1118 // TRestStringOutput cout;
1119 // cout.setcolor(COLOR_BOLDWHITE);
1120 // cout.setorientation(0);
1121 // cout.setborder("|");
1122 // CursorDown(1);
1123
1124 double speedbyte = GetReadingSpeed();
1125
1126 double progsum = 0;
1127 for (auto& n : progAdded) progsum += n;
1128 double progspeed = progsum / ncalculated / printInterval * 1000000;
1129
1130 double prog = 0;
1131 if (fRunInfo->GetFeminosDaqTotalEvents() > 0) {
1132 prog = fProcessedEvents / (double)fRunInfo->GetFeminosDaqTotalEvents() * 100;
1133 } else if (fEventsToProcess == REST_MAXIMUM_EVENTS && fRunInfo->GetFileProcess() != nullptr)
1134 // Nevents is unknown, reading external data file
1135 {
1136 prog = fRunInfo->GetBytesRead() / (double)fRunInfo->GetTotalBytes() * 100;
1137 } else if (fRunInfo->GetFileProcess() != nullptr)
1138 // Nevents is known, reading external data file
1139 {
1140 prog = fProcessedEvents / (double)fEventsToProcess * 100;
1141 } else if (fEventsToProcess == REST_MAXIMUM_EVENTS)
1142 // Nevents is unknown, reading root file
1143 {
1144 prog = fRunInfo->GetCurrentEntry() / (double)inputTreeEntries * 100;
1145 } else {
1146 prog = fProcessedEvents / (double)fEventsToProcess * 100;
1147 }
1148
1149 char* buffer = new char[500]();
1150 if (fRunInfo->GetFileProcess() != nullptr && speedbyte > 0) {
1151 sprintf(buffer, "%d Events (%.1fMB/s), ", fProcessedEvents, speedbyte / 1024 / 1024);
1152 } else {
1153 sprintf(buffer, "%d Events, ", fProcessedEvents);
1154 }
1155
1156 string s1(buffer);
1157
1158 if (fProcStatus == kNormal) {
1159 sprintf(buffer, "%.1f min ETA, (Pause: \"p\") ", (100 - progressLast) / progspeed / 60);
1160 } else {
1161 sprintf(buffer, "%.1f min ETA, (Pause Disabled) ", (100 - progressLast) / progspeed / 60);
1162 }
1163 string s2(buffer);
1164
1165 int barlength = 0;
1166 if (REST_Display_CompatibilityMode) {
1167 barlength = 50;
1168 } else {
1169 barlength = Console::GetWidth() - s1.size() - s2.size() - 9;
1170 }
1171 sprintf(buffer, ("%.1f%[" + MakeProgressBar(prog, barlength) + "]").c_str(), prog);
1172 string s3(buffer);
1173
1174 delete[] buffer;
1175
1176 if (REST_Display_CompatibilityMode) {
1177 if (((int)prog) != progressLastPrinted) {
1178 cout << s1 << s2 << s3 << endl;
1179 progressLastPrinted = (int)prog;
1180 }
1182 printf("%s", (s1 + s2 + s3 + "\r").c_str());
1183 fflush(stdout);
1184 }
1185
1186 bytesAdded[positionCalculated] = fRunInfo->GetBytesRead() - bytesReadLast;
1187 bytesReadLast = fRunInfo->GetBytesRead();
1188 progAdded[positionCalculated] = prog - progressLast;
1189 progressLast = prog;
1190
1191 positionCalculated++;
1192 if (positionCalculated >= ncalculated) positionCalculated -= ncalculated;
1193 }
1194}
1195
1199string TRestProcessRunner::MakeProgressBar(int progress100, int length) {
1200 string progressbar(length, '-');
1201 int n = (double)progress100 / 100 * length;
1202 if (n < 0) n = 0;
1203 if (n > length + 1) n = length + 1;
1204 for (int i = 0; i < n; i++) {
1205 progressbar[i] = '=';
1206 }
1207 if (n < length + 1) progressbar[n] = '>';
1208 return progressbar;
1209}
1210
1211TRestEvent* TRestProcessRunner::GetInputEvent() { return fRunInfo->GetInputEvent(); }
1212
1213TRestAnalysisTree* TRestProcessRunner::GetInputAnalysisTree() { return fRunInfo->GetAnalysisTree(); }
1214
1217
1218 string status;
1219 if (fProcStatus == kNormal)
1220 status = "Normal";
1221 else if (fProcStatus == kStopping)
1222 status = "Terminated";
1223 else
1224 status = "Unknown";
1225
1226 RESTMetadata << "Status : " << status << RESTendl;
1227 RESTMetadata << "Processed events : " << fProcessedEvents << RESTendl;
1228 RESTMetadata << "Analysis tree branches : " << fNBranches << RESTendl;
1229 RESTMetadata << "Thread number : " << fThreadNumber << RESTendl;
1230 RESTMetadata << "Processes in each thread : " << fProcessNumber << RESTendl;
1231 RESTMetadata << "File auto split size: " << fFileSplitSize << RESTendl;
1232 RESTMetadata << "File compression level: " << fFileCompression << RESTendl;
1233 RESTMetadata << "******************************************" << RESTendl;
1234 RESTMetadata << RESTendl;
1235 RESTMetadata << RESTendl;
1236}
REST core data-saving helper based on TTree.
A base class for any REST event process.
void SetRunInfo(TRestRun *r)
Set TRestRun for this process.
Bool_t singleThreadOnly() const
Return whether this process is single std::thread only.
Bool_t isExternal() const
Return whether this process is external process.
void SetParallelProcess(TRestEventProcess *p)
Add parallel process to this process.
A base class for any REST event.
Definition TRestEvent.h:38
virtual void Initialize()=0
std::map< std::string, std::string > fVariables
Saving a list of rml variables. name-value std::pair.
virtual void PrintMetadata()
Implemented it in the derived metadata class to print out specific metadata information.
endl_t RESTendl
Termination flag object for TRestStringOutput.
TiXmlElement * fElementGlobal
Saving the global element, to be passed to the resident class, if necessary.
TRestStringOutput::REST_Verbose_Level fVerboseLevel
Verbose level used to print debug info.
void SetHostmgr(TRestManager *m)
Set the host manager for this class.
TRestStringOutput::REST_Verbose_Level GetVerboseLevel()
returns the verboselevel in type of REST_Verbose_Level enumerator
Int_t LoadConfigFromElement(TiXmlElement *eSectional, TiXmlElement *eGlobal, std::map< std::string, std::string > envs={})
Main starter method.
void SetConfigFile(std::string configFilename)
set config file path from external
void ReadAllParameters()
Reflection methods, Set value of a datamember in class according to TRestMetadata::fElement.
TRestManager * fHostmgr
All metadata classes can be initialized and managed by TRestManager.
std::string fConfigFileName
Full name of the rml file.
virtual Int_t Write(const char *name=nullptr, Int_t option=0, Int_t bufsize=0)
overwriting the write() method with fStore considered
std::string GetParameter(std::string parName, TiXmlElement *e, TString defaultValue=PARAMETER_NOT_FOUND_STR)
Returns the value for the parameter named parName in the given section.
void SetVerboseLevel(TRestStringOutput::REST_Verbose_Level v)
sets the verbose level
Running the processes efficiently with fantastic display.
TString fOutputDataFileName
the TFile pointer being used
void ReadProcInfo()
Create a process info list which used called by TRestRun::FormFormat().
void PrintProcessedEvents(Int_t rateE)
Print number of events processed, file read speed, ETA and a progress bar.
void ResetRunTimes()
Reset running time count to 0.
void BeginOfInit()
Reads information from rml config file.
Int_t ReadConfig(const std::string &keydeclare, TiXmlElement *e)
method to deal with iterated child elements
void MergeOutputFile()
Calls TRestRun::MergeOutputFile() to merge the main file with process's tmp file.
void RunProcess()
The main executer of event process.
void EndOfInit()
Ending of the startup procedure.
TRestEventProcess * InstantiateProcess(TString type, TiXmlElement *ele)
InstantiateProcess in sequential start up.
void PauseMenu()
A pause menu providing some functions during the process.
void Initialize() override
REST run class.
std::string MakeProgressBar(int progress100, int length=100)
Make a string of progress bar with given length and percentage.
void WriteProcessesMetadata(TFile *destination=nullptr)
Write process metadata to the requested file, or fOutputDataFile by default.
Int_t GetNextevtFunc(TRestEvent *targetevt, TRestAnalysisTree *targettree)
Get next event and copy it to the address of targetevt.
void PrintMetadata() override
Implemented it in the derived metadata class to print out specific metadata information.
Bool_t fUseTestRun
Number of files being split.
void FillThreadEventFunc(TRestThread *t)
Calling back the FillEvent() method in TRestThread.
void ConfigOutputFile()
Forming an output file.
static TRestRootFileHandle Open(const std::string &filename, TRestRootFileMode mode)
TRestEvent * GetInputEvent() const
Calling GetInputEvent() will return a basic TRestEvent*.
Definition TRestRun.h:158
REST_Verbose_Level
Enumerate of verbose level, containing five levels.
@ REST_Essential
+show some essential information, as well as warnings
@ REST_Info
+show most of the information for each steps
@ REST_Debug
+show the defined debug messages
@ REST_Silent
show minimum information of the software, as well as error messages
Threaded worker of a process chain.
Definition TRestThread.h:24
static std::pair< std::string, std::string > SeparatePathAndName(const std::string &fullname)
Separate path and filename in a full path+filename string, returns a pair of string.
static bool fileExists(const std::string &filename)
Returns true if the file (or directory) with path filename exists.
static bool isPathWritable(const std::string &path)
Returns true if the path given by argument is writable.
TRestReflector Assembly(const std::string &typeName)
Assembly an object of type: typeName, returning the allocated memory address and size.
std::vector< std::string > Split(std::string in, std::string separator, bool allowBlankString=false, bool removeWhiteSpaces=false, int startPos=-1)
Split the input string according to the given separator. Returning a vector of fragments.
std::string ToUpper(std::string in)
Convert string to its upper case. Alternative of TString::ToUpper.
Int_t StringToInteger(std::string in)
Gets an integer from a string.
std::string ToDateTimeString(time_t time)
Format time_t into string.