From 3b5f2807f461f7a9547ca7459d5e138a35d3ff71 Mon Sep 17 00:00:00 2001 From: Marco Wang Date: Sun, 1 Dec 2019 19:12:06 +0800 Subject: [PATCH 1/3] src/logging.cc: encapsulate log cleaner --- src/logging.cc | 221 ++++++++++++++++++++++++++++--------------------- 1 file changed, 125 insertions(+), 96 deletions(-) diff --git a/src/logging.cc b/src/logging.cc index 42cfd01..12579f6 100644 --- a/src/logging.cc +++ b/src/logging.cc @@ -457,6 +457,29 @@ class LogFileObject : public base::Logger { bool CreateLogfile(const string& time_pid_string); }; +// Encapsulate all log cleaner related states +class LogCleaner { + public: + LogCleaner(); + virtual ~LogCleaner() {} + + void Enable(int overdue_days); + void Disable(); + void Run() const; + + inline bool enabled() const { return enabled_; } + + private: + bool IsLogFromCurrentProject(const string& filename) const; + bool IsLogLastModifiedOver(const string& filepath, int days) const; + vector GetOverdueLogNames(string log_directory, int days) const; + + bool enabled_; + int overdue_days_; +}; + +LogCleaner log_cleaner; + } // namespace class LogDestination { @@ -839,86 +862,6 @@ void LogDestination::DeleteLogDestinations() { sinks_ = NULL; } -namespace { - -bool IsGlogLog(const string& filename) { - // Check if filename matches the pattern of a glog file: - // "...log...". - const int kKeywordCount = 4; - std::string keywords[kKeywordCount] = { - glog_internal_namespace_::ProgramInvocationShortName(), - LogDestination::hostname(), - MyUserName(), - "log" - }; - - int start_pos = 0; - for (int i = 0; i < kKeywordCount; i++) { - if (filename.find(keywords[i], start_pos) == filename.npos) { - return false; - } - start_pos += keywords[i].size() + 1; - } - return true; -} - -bool LastModifiedOver(const string& filepath, int days) { - // Try to get the last modified time of this file. - struct stat file_stat; - - if (stat(filepath.c_str(), &file_stat) == 0) { - // A day is 86400 seconds, so 7 days is 86400 * 7 = 604800 seconds. - time_t last_modified_time = file_stat.st_mtime; - time_t current_time = time(NULL); - return difftime(current_time, last_modified_time) > days * 86400; - } - - // If failed to get file stat, don't return true! - return false; -} - -vector GetOverdueLogNames(string log_directory, int days) { - // The names of overdue logs. - vector overdue_log_names; - - // Try to get all files within log_directory. - DIR *dir; - struct dirent *ent; - - char dir_delim = '/'; -#ifdef OS_WINDOWS - dir_delim = '\\'; -#endif - - // If log_directory doesn't end with a slash, append a slash to it. - if (log_directory.at(log_directory.size() - 1) != dir_delim) { - log_directory += dir_delim; - } - - if ((dir=opendir(log_directory.c_str()))) { - while ((ent=readdir(dir))) { - if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) { - continue; - } - string filepath = log_directory + ent->d_name; - if (IsGlogLog(ent->d_name) && LastModifiedOver(filepath, days)) { - overdue_log_names.push_back(filepath); - } - } - closedir(dir); - } - - return overdue_log_names; -} - -// Is log_cleaner enabled? -// This option can be enabled by calling google::EnableLogCleaner(days) -bool log_cleaner_enabled_; -int log_cleaner_overdue_days_ = 7; - -} // namespace - - namespace { LogFileObject::LogFileObject(LogSeverity severity, @@ -1269,18 +1212,110 @@ void LogFileObject::Write(bool force_flush, } #endif // Perform clean up for old logs - if (log_cleaner_enabled_) { - const vector& dirs = GetLoggingDirectories(); - for (size_t i = 0; i < dirs.size(); i++) { - vector logs = GetOverdueLogNames(dirs[i], log_cleaner_overdue_days_); - for (size_t j = 0; j < logs.size(); j++) { - static_cast(unlink(logs[j].c_str())); - } - } + if (log_cleaner.enabled()) { + log_cleaner.Run(); } } } + +LogCleaner::LogCleaner() : enabled_(false), overdue_days_(7) {} + +void LogCleaner::Enable(int overdue_days) { + // Setting overdue_days to 0 day should not be allowed! + // Since all logs will be deleted immediately, which will cause troubles. + assert(overdue_days > 0); + + enabled_ = true; + overdue_days_ = overdue_days; +} + +void LogCleaner::Disable() { + enabled_ = false; +} + +void LogCleaner::Run() const { + assert(enabled_ && overdue_days_ > 0); + + const vector& dirs = GetLoggingDirectories(); + for (size_t i = 0; i < dirs.size(); i++) { + vector logs = GetOverdueLogNames(dirs[i], overdue_days_); + for (size_t j = 0; j < logs.size(); j++) { + static_cast(unlink(logs[j].c_str())); + } + } +} + +bool LogCleaner::IsLogFromCurrentProject(const string& filename) const { + // Check if filename matches the pattern of a glog file: + // "...log...". + const int kKeywordCount = 4; + std::string keywords[kKeywordCount] = { + glog_internal_namespace_::ProgramInvocationShortName(), + LogDestination::hostname(), + MyUserName(), + "log" + }; + + int start_pos = 0; + for (int i = 0; i < kKeywordCount; i++) { + if (filename.find(keywords[i], start_pos) == filename.npos) { + return false; + } + start_pos += keywords[i].size() + 1; + } + return true; +} + +bool LogCleaner::IsLogLastModifiedOver(const string& filepath, int days) const { + // Try to get the last modified time of this file. + struct stat file_stat; + + if (stat(filepath.c_str(), &file_stat) == 0) { + // A day is 86400 seconds, so 7 days is 86400 * 7 = 604800 seconds. + time_t last_modified_time = file_stat.st_mtime; + time_t current_time = time(NULL); + return difftime(current_time, last_modified_time) > days * 86400; + } + + // If failed to get file stat, don't return true! + return false; +} + +vector LogCleaner::GetOverdueLogNames(string log_directory, int days) const { + // The names of overdue logs. + vector overdue_log_names; + + // Try to get all files within log_directory. + DIR *dir; + struct dirent *ent; + + char dir_delim = '/'; +#ifdef OS_WINDOWS + dir_delim = '\\'; +#endif + + // If log_directory doesn't end with a slash, append a slash to it. + if (log_directory.at(log_directory.size() - 1) != dir_delim) { + log_directory += dir_delim; + } + + if ((dir=opendir(log_directory.c_str()))) { + while ((ent=readdir(dir))) { + if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) { + continue; + } + string filepath = log_directory + ent->d_name; + if (IsLogFromCurrentProject(ent->d_name) && IsLogLastModifiedOver(filepath, days)) { + overdue_log_names.push_back(filepath); + } + } + closedir(dir); + } + + return overdue_log_names; +} + } // namespace // Static log data space to avoid alloc failures in a LOG(FATAL) @@ -2319,17 +2354,11 @@ void ShutdownGoogleLogging() { } void EnableLogCleaner(int overdue_days) { - log_cleaner_enabled_ = true; - - // Setting overdue_days to 0 day should not be allowed! - // Since all logs will be deleted immediately, which will cause troubles. - if (overdue_days > 0) { - log_cleaner_overdue_days_ = overdue_days; - } + log_cleaner.Enable(overdue_days); } void DisableLogCleaner() { - log_cleaner_enabled_ = false; + log_cleaner.Disable(); } _END_GOOGLE_NAMESPACE_ From 7764e4a8b469a1db760b85ab04d7f938d7581574 Mon Sep 17 00:00:00 2001 From: Marco Wang Date: Sun, 1 Dec 2019 22:06:12 +0800 Subject: [PATCH 2/3] src/logging.cc: match logfile with base_filename --- src/logging.cc | 135 +++++++++++++++++++++++++++---------------------- 1 file changed, 75 insertions(+), 60 deletions(-) diff --git a/src/logging.cc b/src/logging.cc index 12579f6..291b658 100644 --- a/src/logging.cc +++ b/src/logging.cc @@ -465,17 +465,19 @@ class LogCleaner { void Enable(int overdue_days); void Disable(); - void Run() const; + void Run(bool base_filename_selected, const string& base_filename) const; inline bool enabled() const { return enabled_; } private: - bool IsLogFromCurrentProject(const string& filename) const; + vector GetOverdueLogNames(string log_directory, int days, + const string& base_filename) const; + bool IsLogFromCurrentProject(const string& filepath, const string& base_filename) const; bool IsLogLastModifiedOver(const string& filepath, int days) const; - vector GetOverdueLogNames(string log_directory, int days) const; - + bool enabled_; int overdue_days_; + char dir_delim_; // filepath delimiter ('/' or '\\') }; LogCleaner log_cleaner; @@ -1213,13 +1215,20 @@ void LogFileObject::Write(bool force_flush, #endif // Perform clean up for old logs if (log_cleaner.enabled()) { - log_cleaner.Run(); + if (base_filename_selected_ && base_filename_.empty()) { + return; + } + log_cleaner.Run(base_filename_selected_, base_filename_); } } } -LogCleaner::LogCleaner() : enabled_(false), overdue_days_(7) {} +LogCleaner::LogCleaner() : enabled_(false), overdue_days_(7), dir_delim_('/') { +#ifdef OS_WINDOWS + dir_delim_ = '\\'; +#endif +} void LogCleaner::Enable(int overdue_days) { // Setting overdue_days to 0 day should not be allowed! @@ -1234,37 +1243,77 @@ void LogCleaner::Disable() { enabled_ = false; } -void LogCleaner::Run() const { +void LogCleaner::Run(bool base_filename_selected, const string& base_filename) const { assert(enabled_ && overdue_days_ > 0); - const vector& dirs = GetLoggingDirectories(); + vector dirs; + + if (base_filename_selected) { + string dir = base_filename.substr(0, base_filename.find_last_of(dir_delim_) + 1); + dirs.push_back(dir); + } else { + dirs = GetLoggingDirectories(); + } + for (size_t i = 0; i < dirs.size(); i++) { - vector logs = GetOverdueLogNames(dirs[i], overdue_days_); + vector logs = GetOverdueLogNames(dirs[i], overdue_days_, base_filename); for (size_t j = 0; j < logs.size(); j++) { static_cast(unlink(logs[j].c_str())); } } } -bool LogCleaner::IsLogFromCurrentProject(const string& filename) const { - // Check if filename matches the pattern of a glog file: - // "...log...". - const int kKeywordCount = 4; - std::string keywords[kKeywordCount] = { - glog_internal_namespace_::ProgramInvocationShortName(), - LogDestination::hostname(), - MyUserName(), - "log" - }; +vector LogCleaner::GetOverdueLogNames(string log_directory, int days, + const string& base_filename) const { + // The names of overdue logs. + vector overdue_log_names; - int start_pos = 0; - for (int i = 0; i < kKeywordCount; i++) { - if (filename.find(keywords[i], start_pos) == filename.npos) { - return false; - } - start_pos += keywords[i].size() + 1; + // Try to get all files within log_directory. + DIR *dir; + struct dirent *ent; + + // If log_directory doesn't end with a slash, append a slash to it. + if (log_directory.at(log_directory.size() - 1) != dir_delim_) { + log_directory += dir_delim_; } - return true; + + if ((dir=opendir(log_directory.c_str()))) { + while ((ent=readdir(dir))) { + if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) { + continue; + } + string filepath = log_directory + ent->d_name; + if (IsLogFromCurrentProject(filepath, base_filename) && + IsLogLastModifiedOver(filepath, days)) { + overdue_log_names.push_back(filepath); + } + } + closedir(dir); + } + + return overdue_log_names; +} + +bool LogCleaner::IsLogFromCurrentProject(const string& filepath, + const string& base_filename) const { + // We should remove duplicated delimiters from `base_filename`, e.g., + // before: "/tmp//...log." + // after: "/tmp/...log." + string cleaned_base_filename; + + for (size_t i = 0; i < base_filename.size(); ++i) { + const char& c = base_filename[i]; + if (cleaned_base_filename.empty()) { + cleaned_base_filename += c; + } else if (c != dir_delim_ || + c != cleaned_base_filename.at(cleaned_base_filename.size() - 1)) { + cleaned_base_filename += c; + } + } + + // If the filename of the given logfile starts with `cleaned_base_filename`, + // then this logfile is from current project. + return filepath.find(cleaned_base_filename) == 0; } bool LogCleaner::IsLogLastModifiedOver(const string& filepath, int days) const { @@ -1282,40 +1331,6 @@ bool LogCleaner::IsLogLastModifiedOver(const string& filepath, int days) const { return false; } -vector LogCleaner::GetOverdueLogNames(string log_directory, int days) const { - // The names of overdue logs. - vector overdue_log_names; - - // Try to get all files within log_directory. - DIR *dir; - struct dirent *ent; - - char dir_delim = '/'; -#ifdef OS_WINDOWS - dir_delim = '\\'; -#endif - - // If log_directory doesn't end with a slash, append a slash to it. - if (log_directory.at(log_directory.size() - 1) != dir_delim) { - log_directory += dir_delim; - } - - if ((dir=opendir(log_directory.c_str()))) { - while ((ent=readdir(dir))) { - if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) { - continue; - } - string filepath = log_directory + ent->d_name; - if (IsLogFromCurrentProject(ent->d_name) && IsLogLastModifiedOver(filepath, days)) { - overdue_log_names.push_back(filepath); - } - } - closedir(dir); - } - - return overdue_log_names; -} - } // namespace // Static log data space to avoid alloc failures in a LOG(FATAL) From 36fa99ba2c90603b5c5a39173d8cf2801d581cad Mon Sep 17 00:00:00 2001 From: Marco Wang Date: Mon, 9 Dec 2019 08:48:41 +0800 Subject: [PATCH 3/3] src/logging.cc: check the entire filename format --- src/logging.cc | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/src/logging.cc b/src/logging.cc index 291b658..9724ae4 100644 --- a/src/logging.cc +++ b/src/logging.cc @@ -1297,12 +1297,13 @@ vector LogCleaner::GetOverdueLogNames(string log_directory, int days, bool LogCleaner::IsLogFromCurrentProject(const string& filepath, const string& base_filename) const { // We should remove duplicated delimiters from `base_filename`, e.g., - // before: "/tmp//...log." - // after: "/tmp/...log." + // before: "/tmp//.." + // after: "/tmp/.." string cleaned_base_filename; for (size_t i = 0; i < base_filename.size(); ++i) { const char& c = base_filename[i]; + if (cleaned_base_filename.empty()) { cleaned_base_filename += c; } else if (c != dir_delim_ || @@ -1311,9 +1312,34 @@ bool LogCleaner::IsLogFromCurrentProject(const string& filepath, } } - // If the filename of the given logfile starts with `cleaned_base_filename`, - // then this logfile is from current project. - return filepath.find(cleaned_base_filename) == 0; + // Return early if the filename doesn't start with `cleaned_base_filename`. + if (filepath.find(cleaned_base_filename) != 0) { + return false; + } + + // The characters after `cleaned_base_filename` should match the format: + // YYYYMMDD-HHMMSS.pid + for (size_t i = cleaned_base_filename.size(); i < filepath.size(); i++) { + const char& c = filepath[i]; + + if (i <= cleaned_base_filename.size() + 7) { // 0 ~ 7 : YYYYMMDD + if (c < '0' || c > '9') { return false; } + + } else if (i == cleaned_base_filename.size() + 8) { // 8: - + if (c != '-') { return false; } + + } else if (i <= cleaned_base_filename.size() + 14) { // 9 ~ 14: HHMMSS + if (c < '0' || c > '9') { return false; } + + } else if (i == cleaned_base_filename.size() + 15) { // 15: . + if (c != '.') { return false; } + + } else if (i >= cleaned_base_filename.size() + 16) { // 16+: pid + if (c < '0' || c > '9') { return false; } + } + } + + return true; } bool LogCleaner::IsLogLastModifiedOver(const string& filepath, int days) const {