General Purpose Geodetic Library
SgVlbiSessionUtils4LogProc.cpp
Go to the documentation of this file.
1 /*
2  *
3  * This file is a part of Space Geodetic Library. The library is used by
4  * nuSolve, a part of CALC/SOLVE system, and designed to make analysis of
5  * geodetic VLBI observations.
6  * Copyright (C) 2010-2020 Sergei Bolotin.
7  *
8  * This program is free software: you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation, either version 3 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  * GNU General Public License for more details.
17  *
18  * You should have received a copy of the GNU General Public License
19  * along with this program. If not, see <http://www.gnu.org/licenses/>.
20  *
21  */
22 
23 #include <iostream>
24 #include <stdlib.h>
25 
26 
27 #include <QtCore/QDir>
28 #include <QtCore/QFile>
29 #include <QtCore/QList>
30 
31 
32 #include <SgVlbiSession.h>
33 
34 #include <SgLogger.h>
35 #include <SgNetworkStnRecord.h>
36 #include <SgStnLogCollector.h>
37 
38 
39 
40 
41 
42 
43 
44 
45 
46 /*=======================================================================================================
47 *
48 * METHODS:
49 *
50 *======================================================================================================*/
51 //
52 bool SgVlbiSession::importDataFromLogFiles(const QString& path2sessions,
53  const QList<QString>& stns, const QList<QString>& kinds,
54  const QMap<QString, int>* defaultCableSignByStn,
55  const QMap<QString, QString>* rinexFileNameByStn,
56  const QMap<QString, double>* rinexPressureOffsetByStn,
57  bool have2overwriteAntabFile, bool reportAllTsysData,
58  const QString& altPath2InputFiles,
59  const QString& orderOfMeteo)
60 {
61  QMap<QString, QString> logFileByKey;
62  QMap<QString, QString> cdmsFileByKey;
63  QMap<QString, QString> pcmtFileByKey;
64  QMap<QString, QString> metFileByKey;
66  QString path2logFiles("");
67  QString sessCode(sessionCode_.toLower());
68  QString str("");
69  QString stnKey("");
70  int nYear=tStart_.calcYear();
71  bool haveFoundSomething=false;
72  bool haveVlbaLog=false;
73  bool doLogs=true, doCdms=true, doPcmt=true, doMets=true;
75  ns_codes.readFile();
76  //
77  if (kinds.size())
78  {
79  doLogs = doCdms = doPcmt = doMets = false;
80  for (int i=0; i<kinds.size(); i++)
81  if (kinds.at(i) == "log")
82  doLogs = true;
83  else if (kinds.at(i) == "cdms")
84  doCdms = true;
85  else if (kinds.at(i) == "pcmt")
86  doPcmt = true;
87  else if (kinds.at(i) == "met")
88  doMets = true;
89  };
90  //
91  if (altPath2InputFiles.size())
92  path2logFiles = altPath2InputFiles;
93  else
94  path2logFiles = path2sessions + "/" + QString("").sprintf("%04d", nYear) + "/" + sessCode + "/";
95  //
97  "::importDataFromLogFiles(): checking [" + path2logFiles + "] directory for station log files");
98  // reset the flag:
100  //
101  //
102  if (stns.size())
103  {
104  for (int i=0; i<stns.size(); i++)
105  {
106  stnInpt2Key(stns.at(i), ns_codes, stnKey);
107  if (stnKey.size())
108  {
109  if (stationsByName_.contains(stnKey))
110  stations.append(stationsByName_.value(stnKey));
111  else
113  "::importDataFromLogFiles(): cannot find station " + stnKey + " in the session");
114  };
115  };
116  }
117  else
118  for (StationsByName_it it=stationsByName_.begin(); it!=stationsByName_.end(); ++it)
119  stations.append(it.value());
120  //
121  //
122  //
124  "::importDataFromLogFiles(): collected " + QString("").setNum(stations.size()) +
125  " stations to process");
126  //
127  QDir dir(path2logFiles);
128  QStringList patterns;
129  QStringList fileList;
130  for (int i=0; i<stations.size(); i++)
131  {
132  SgVlbiStationInfo *stn=stations.at(i);
133  //
134  if (ns_codes.recsByName().contains(stn->getKey()))
135  {
136  // can be more than one in the list, check it later:
137  const SgNetworkStnRecord &rec=ns_codes.recsByName().find(stn->getKey()).value();
138  if (rec.isValid())
139  {
140  const QString& sCode=rec.getCode();
141  stn->setSid(sCode.at(0).toLatin1(), sCode.at(1).toLatin1());
142  //
143  if (doLogs)
144  {
145  str = path2logFiles + sessCode + sCode.toLower() + ".log";
146  if (QFile::exists(str))
147  logFileByKey[stn->getKey()] = str;
148  };
149  //
150  //
151  // VGOS produced data:
152  // CDMS:
153  if (doCdms)
154  {
155  str = path2logFiles + sessCode + sCode.toLower() + ".cdms.dat";
156  if (QFile::exists(str))
157  cdmsFileByKey[stn->getKey()] = str;
158  };
159  //
160  //
161  // PCMT:
162  if (doPcmt)
163  {
164  str = sessCode + sCode.toLower() + ".pcmt.*.*.dat";
165  patterns.clear();
166  patterns << str;
167  fileList = dir.entryList(patterns,
168  QDir::Files | QDir::NoDotAndDotDot | QDir::Readable, QDir::Time);
169  if (fileList.size())
170  {
171  str = fileList.at(0);
172  pcmtFileByKey[stn->getKey()] = path2logFiles + str;
173  // complain:
174  if (fileList.size() > 1)
176  "::importDataFromLogFiles(): more than one PCMT file found for the station " +
177  stn->name() + ", the last modified will be used: " + path2logFiles + str);
178  };
179  };
180  //
181  //
182  // external meteo data:
183  if (doMets)
184  {
185  str = path2logFiles + sessCode + sCode.toLower() + ".met.dat";
186  if (QFile::exists(str))
187  metFileByKey[stn->getKey()] = str;
188  };
189  }
190  else
192  "::importDataFromLogFiles(): the ns-codes.txt record for the station " + stn->name() +
193  " is not valid");
194  }
195  else
197  "::importDataFromLogFiles(): cannot find station " + stn->name() +
198  " in the ns-codes.txt file");
199  };
200  //
201  //
202  //
203  // check for existance of a VLBA log file:
204  SgVlbaLogCollector vlbaLogCollector;
205  str = path2logFiles + sessCode + "cal.vlba";
207  "::importDataFromLogFiles(): checking " + str + " log file for VLBA stations");
208  if (QFile::exists(str))
209  {
211  "::importDataFromLogFiles(): found the log file for VLBA stations");
212  haveVlbaLog = vlbaLogCollector.readLogFile(str, nYear);
213  }
214  else
216  "::importDataFromLogFiles(): the log file for VLBA stations was not found");
217  //
218  //
219  //
220  if (logFileByKey.size() != stationsByName_.size()) // something missed, make additional search:
221  search4missedLogFiles(logFileByKey, ns_codes);
222  //
223  //
224  //
225  if (logFileByKey.size())
227  "::importDataFromLogFiles(): collected " + QString("").setNum(logFileByKey.size()) +
228  " log file" + QString(logFileByKey.size()>1?"s":""));
229  if (cdmsFileByKey.size())
231  "::importDataFromLogFiles(): collected " + QString("").setNum(cdmsFileByKey.size()) +
232  " CDMS file" + QString(cdmsFileByKey.size()>1?"s":""));
233  if (pcmtFileByKey.size())
235  "::importDataFromLogFiles(): collected " + QString("").setNum(pcmtFileByKey.size()) +
236  " PCMT file" + QString(pcmtFileByKey.size()>1?"s":""));
237  if (metFileByKey.size())
239  "::importDataFromLogFiles(): collected " + QString("").setNum(metFileByKey.size()) +
240  " file" + QString(metFileByKey.size()>1?"s":"") + " with external meteorological data");
241  //
242  //
243  //
244  // First, check for aux (vgos) cal files:
245  //
246  // CDMS:
247  for (QMap<QString, QString>::iterator it=cdmsFileByKey.begin(); it!=cdmsFileByKey.end(); ++it)
248  {
249  SgVlbiStationInfo *stn=stationsByName_.value(it.key());
251  {
253  "::importDataFromLogFiles(): processing " + it.value() +
254  ", a CDMS file for " + stn->name() + " station");
255  SgStnLogCollector logCollector;
256  if (logCollector.readLogFile(it.value(), stn->name(), tStart_, tFinis_, ""))
257  {
258  if (logCollector.propagateData(stn, false, have2overwriteAntabFile, false))
259  {
261  "::importDataFromLogFiles(): data propagation successfully completed for " + stn->name());
262  haveFoundSomething = true;
264  {
265  stn->setCableCalsOriginTxt("Data extracted from the file " + it.value());
267  };
268  }
269  else
271  "::importDataFromLogFiles(): data propagation has failed for " + stn->name());
272  }
273  else
275  "::importDataFromLogFiles(): reading of " + it.value() + " failed");
276  }
277  else
279  "::importDataFromLogFiles(): the station " + stn->name() +
280  " already has cable calibration data, the file " + it.value() + " is skipped");
281  };
282  //
283  // PCMT:
284  for (QMap<QString, QString>::iterator it=pcmtFileByKey.begin(); it!=pcmtFileByKey.end(); ++it)
285  {
286  SgVlbiStationInfo *stn=stationsByName_.value(it.key());
288  {
290  "::importDataFromLogFiles(): processing " + it.value() +
291  ", a PCMT file for " + stn->name() + " station");
292  SgStnLogCollector logCollector;
293  if (logCollector.readLogFile(it.value(), stn->name(), tStart_, tFinis_, ""))
294  {
295  if (logCollector.propagateData(stn, false, have2overwriteAntabFile, false))
296  {
298  "::importDataFromLogFiles(): data propagation successfully completed for " + stn->name());
299  haveFoundSomething = true;
301  {
302  stn->setCableCalsOriginTxt("Data extracted from the file " + it.value());
304  };
305  }
306  else
308  "::importDataFromLogFiles(): data propagation has failed for " + stn->name());
309  }
310  else
312  "::importDataFromLogFiles(): reading of " + it.value() + " failed");
313  }
314  else
316  "::importDataFromLogFiles(): the station " + stn->name() +
317  " already has cable calibration data, the file " + it.value() + " is skipped");
318  };
319  //
320  // External meteo data:
321  for (QMap<QString, QString>::iterator it=metFileByKey.begin(); it!=metFileByKey.end(); ++it)
322  {
323  SgVlbiStationInfo *stn=stationsByName_.value(it.key());
325  {
327  "::importDataFromLogFiles(): processing " + it.value() +
328  " file with external meteo data for " + stn->name() + " station");
329  SgStnLogCollector logCollector;
330  if (logCollector.readLogFile(it.value(), stn->name(), tStart_, tFinis_, ""))
331  {
332  if (logCollector.propagateData(stn, false, have2overwriteAntabFile, false))
333  {
335  "::importDataFromLogFiles(): data propagation successfully completed for " + stn->name());
336  haveFoundSomething = true;
338  {
339  stn->setMeteoDataOriginTxt("Data extracted from the file " + it.value());
341  };
342  }
343  else
345  "::importDataFromLogFiles(): data propagation has failed for " + stn->name());
346  }
347  else
349  "::importDataFromLogFiles(): reading of " + it.value() + " failed");
350  }
351  else
353  "::importDataFromLogFiles(): the station " + stn->name() +
354  " already has meteo data, the file " + it.value() + " is skipped");
355  };
356  //
357  //
358  SgChannelSkeded psf;
359  psf.parseSkdFile(path2logFiles + sessCode + ".skd");
360 
361 
362  // Import data from the existing FS log files of regular stations:
363  for (QMap<QString, QString>::iterator it=logFileByKey.begin(); it!=logFileByKey.end(); ++it)
364  {
365  SgVlbiStationInfo *stn=stationsByName_.value(it.key());
369  {
371  "::importDataFromLogFiles(): processing " + it.value() +
372  ", a log file for " + stn->name() + " station");
373  // clear the attributes:
374  SgStnLogCollector logCollector;
375  // set deafult cable sign map:
376  logCollector.setDefaultCableSignByStn(defaultCableSignByStn);
377  logCollector.setRinexFileNameByStn(rinexFileNameByStn);
378  logCollector.setRinexPressureOffsetByStn(rinexPressureOffsetByStn);
379  logCollector.setChannelSkeded(&psf);
380  // it should speed up a little bit:
381 // logCollector.setAntcalOutputData(AOD_TRADITIONAL | AOD_TSYS);
382  logCollector.setAntcalOutputData(AOD_TRADITIONAL);
383 //
384 // 4 debug:
385 // logCollector.setUseSkededChannelSetup(true);
386  //
387  //
388  if (logCollector.readLogFile(it.value(), stn->name(), tStart_, tFinis_, orderOfMeteo))
389  {
390  if (logCollector.propagateData(stn,
391  // true,
392  have2overwriteAntabFile, // temporary
393  have2overwriteAntabFile, reportAllTsysData))
394  {
396  "::importDataFromLogFiles(): data propagation successfully completed for " + stn->name());
397  haveFoundSomething = true;
399  {
400  stn->setCableCalsOriginTxt("Data extracted from the file " + it.value());
403  else
405  };
407  {
408  stn->setMeteoDataOriginTxt("Data extracted from the file " + it.value());
410  };
411  }
412  else
414  "::importDataFromLogFiles(): data propagation has failed for " + stn->name());
415  }
416  else
418  "::importDataFromLogFiles(): reading of " + it.value() + " failed");
419  }
420  else
422  "::importDataFromLogFiles(): the station " + stn->name() +
423  " already has all data, the file " + it.value() + " is skipped");
424  };
425  //
426  //
427  //
428  //
429  //
430  // import data from the VLBA log file, if it is exists:
431  if (haveVlbaLog)
432  for (StationsByName_it it=stationsByName_.begin(); it!=stationsByName_.end(); ++it)
433  {
434  SgVlbiStationInfo *stn=it.value();
435  if (vlbaLogCollector.isVlba(stn->getKey()))
436  {
437  if (vlbaLogCollector.propagateData(stn))
438  {
440  "::importDataFromLogFiles(): data propagation successfully completed for " + stn->name());
441  haveFoundSomething = true;
443  {
444  stn->setCableCalsOriginTxt("Data extracted from the file " +
445  vlbaLogCollector.getLogFileName());
447  };
449  {
450  stn->setMeteoDataOriginTxt("Data extracted from the file " +
451  vlbaLogCollector.getLogFileName());
453  };
454  }
455  else
457  "::importDataFromLogFiles(): data propagation has failed for " + stn->name());
458  };
459  };
460  //
461  if (haveFoundSomething)
463  //
464  //
466  "::importDataFromLogFiles(): import of data from station log files has been finished, " +
467  (haveFoundSomething?"got some new information":"nothing useful found"));
468  return haveFoundSomething;
469 };
470 
471 
472 
473 
474 
475 //
476 void SgVlbiSession::search4missedLogFiles(QMap<QString, QString>& /*logFileByKey*/,
477  const SgNetworkStations &/*ns_codes*/)
478 {
479 
480 };
481 
482 
483 
484 //
485 bool SgVlbiSession::resetDataFromLogFiles(bool doCable, bool doMeteo, bool doTsys,
486  const QList<QString>& stns)
487 {
488  if (!(doCable || doMeteo || doTsys))
489  return false;
490  //
491  //
492  QList<SgVlbiStationInfo*> stations;
494  bool ns_codes_read(false);
495  QString stnKey("");
496  //
497  if (stns.size())
498  {
499  for (int i=0; i<stns.size(); i++)
500  {
501  if (stns.at(i).size()==2 && !ns_codes_read) // do not need to read the file if it is not a 2-char id
502  {
503  ns_codes.readFile();
504  ns_codes_read = true;
505  };
506  stnInpt2Key(stns.at(i), ns_codes, stnKey);
507  if (stnKey.size())
508  {
509  if (stationsByName_.contains(stnKey))
510  stations.append(stationsByName_.value(stnKey));
511  else
513  "::resetDataFromLogFiles(): cannot find station " + stnKey + " in the session");
514  };
515  };
516  }
517  else
518  for (StationsByName_it it=stationsByName_.begin(); it!=stationsByName_.end(); ++it)
519  stations.append(it.value());
520  //
521  //
522  if (!stations.size())
523  {
525  "::resetDataFromLogFiles(): no station found to process");
526  return false;
527  }
528  else
530  "::resetDataFromLogFiles(): collected " + QString("").setNum(stations.size()) +
531  " stations to process");
532  //
533  //
534  //
535  for (int i=0; i<stations.size(); i++)
536  {
537  SgVlbiStationInfo *stn=stations.at(i);
538  if (doCable)
539  {
540  stn->resetCable();
542  "::resetDataFromLogFiles(): the cable calibration corrections were set to zeros for station " +
543  stn->getKey());
544  };
545  if (doMeteo)
546  {
547  stn->resetMeteo();
549  "::resetDataFromLogFiles(): the meteorological data were set to zeros for station " +
550  stn->getKey());
551  };
552  if (doTsys)
553  {
554  stn->resetTsys();
556  "::resetDataFromLogFiles(): the meteorological data were set to zeros for station " +
557  stn->getKey());
558  };
559  };
561  return true;
562 };
563 
564 
565 
566 //
567 void SgVlbiSession::stnInpt2Key(const QString& stnNameInpt, SgNetworkStations& ns_codes, QString& key)
568 {
569  QString str(stnNameInpt);
570  key = "";
571  if (str.size() == 2) // have to check ns-codes:
572  {
573  if (ns_codes.recsById().contains(str))
574  {
575  key = ns_codes.recsById().value(str).getName();
577  "::stnInpt2Key(): found a station name " + key + " for the code \"" + str + "\"");
578  }
579  else if (str.at(0).isLower())
580  {
581  str[0] = str.at(0).toUpper();
582  if (ns_codes.recsById().contains(str))
583  {
584  key = ns_codes.recsById().value(str).getName();
586  "::stnInpt2Key(): found a station name " + key + " for the code \"" + str + "\"");
587  }
588  else if (str.at(1).isUpper())
589  {
590  str[1] = str.at(1).toLower();
591  if (ns_codes.recsById().contains(str))
592  {
593  key = ns_codes.recsById().value(str).getName();
595  "::stnInpt2Key(): found a station name " + key + " for the code \"" + str + "\"");
596  }
597  else
599  "::stnInpt2Key(): cannot find a station name for the code \"" + str + "\"");
600  }
601  else
603  "::stnInpt2Key(): cannot find a station name for the code \"" + str + "\"");
604  }
605  else
607  "::stnInpt2Key(): cannot find a station name for the code \"" + str + "\"");
608  }
609  else if (str.size() > 2)
610  key = stnNameInpt.leftJustified(8, ' ', true);
611 };
612 /*=====================================================================================================*/
613 
614 
615 
616 
617 
618 /*=====================================================================================================*/
619 //
620 // FRIENDS:
621 //
622 /*=====================================================================================================*/
623 //
624 
625 
626 
627 /*=====================================================================================================*/
628 //
629 // aux functions:
630 //
631 
632 /*=====================================================================================================*/
633 
634 /*=====================================================================================================*/
SgLogger * logger
Definition: SgLogger.cpp:231
@ AOD_TRADITIONAL
QMap< QString, SgVlbiStationInfo * >::iterator StationsByName_it
bool isAttr(uint a) const
Definition: SgAttribute.h:226
void delAttr(uint a)
Definition: SgAttribute.h:210
void addAttr(uint a)
Definition: SgAttribute.h:202
bool parseSkdFile(const QString &fileName)
virtual void write(LogLevel, quint32, const QString &, bool=false)
Definition: SgLogger.cpp:88
@ PREPROC
Definition: SgLogger.h:98
int calcYear() const
Definition: SgMJD.cpp:204
const QMultiMap< QString, SgNetworkStnRecord > & recsById() const
const QMultiMap< QString, SgNetworkStnRecord > & recsByName() const
const QString & getCode() const
const QString & getKey() const
Definition: SgObjectInfo.h:319
void setDefaultCableSignByStn(const QMap< QString, int > *map)
void setAntcalOutputData(int outputData)
void setRinexPressureOffsetByStn(const QMap< QString, double > *map)
bool propagateData(SgVlbiStationInfo *stn, bool createAntabFile, bool overwriteAntabFile, bool reportAllTsysData)
void setRinexFileNameByStn(const QMap< QString, QString > *map)
void setChannelSkeded(SgChannelSkeded *cs)
bool readLogFile(const QString &fileName, const QString &stnName, const SgMJD &tFirst, const SgMJD &tLast, const QString &orderOfMeteo)
const QString & getLogFileName() const
bool readLogFile(const QString &fileName, int year)
bool propagateData(SgVlbiStationInfo *stn)
bool isVlba(const QString &key) const
@ Attr_FF_AUX_OBS_MODIFIED
station log related data were modified; //vgosDbProcLogs
SgMJD tFinis_
last epoch of the observations;
QString sessionCode_
official session code (from Masterfile);
SgMJD tStart_
first epoch of the observations;
StationsByName stationsByName_
void search4missedLogFiles(QMap< QString, QString > &logFileByKey, const SgNetworkStations &ns_codes)
QString path2Masterfile_
bool resetDataFromLogFiles(bool doCable, bool doMeteo, bool doTsys, const QList< QString > &stations)
bool importDataFromLogFiles(const QString &path2sessions, const QList< QString > &stations, const QList< QString > &kinds, const QMap< QString, int > *defaultCableSignByStn, const QMap< QString, QString > *rinexFileNameByStn, const QMap< QString, double > *rinexPressureOffsetByStn, bool have2overwriteAntabFile, bool reportAllTsysData, const QString &altPath2InputFiles, const QString &orderOfMeteo)
static QString className()
void stnInpt2Key(const QString &stnNameInpt, SgNetworkStations &ns_codes, QString &key)
void setMeteoDataOriginTxt(const QString &origin)
QString name() const
@ Attr_HAS_CABLE_CAL
a station has cable calibration readings;
@ Attr_HAS_METEO
a station has meteo parameters;
@ Attr_CABLE_CAL_IS_CDMS
FS log file contains /CDMS/ readings instead of /cable/;.
void setMeteoDataOrigin(MeteoDataOrigin origin)
void setCableCalsOriginTxt(const QString &origin)
void setCableCalsOrigin(CableCalsOrigin origin)
void setSid(char c1, char c2)