JezK
Edit File: slurp_exim_mainlog
#!/usr/local/cpanel/3rdparty/bin/perl # cpanel - scripts/slurp_exim_mainlog Copyright 2022 cPanel, L.L.C. # All rights reserved. # copyright@cpanel.net http://cpanel.net # This code is subject to the cPanel license. Unauthorized copying is prohibited package scripts::slurp_exim_mainlog; use strict; use warnings; use parent qw( Cpanel::HelpfulScript ); use Cpanel::EximStats::ImportInProgress (); use Cpanel::Autodie::Unlink (); use Cpanel::Daemonizer::Tiny (); use Cpanel::Exception (); use Cpanel::EximStats::Retention (); use Cpanel::FileUtils::Dir (); use Cpanel::FileUtils::TouchFile (); use Cpanel::PIDFile (); use Cpanel::Time::ISO (); use Try::Tiny; =encoding utf-8 =head1 NAME scripts::slurp_exim_mainlog =head1 SYNOPSIS slurp_exim_mainlog ( --reimport | --force | --help ) This command will do an import of the unarchived (non-'.gz') /var/log/exim_mainlog* files that are newer than the exim retention setting (90 by default) days. The slurp normally will only run once, but if you pass the --reimport flag it will attempt to import the log files after a 7 day waiting period. If you pass the --force flag, the import will start with no regard to the last time it ran. =head1 DESCRIPTION This script does an import of the exim_mainlog files in /var/log. The import can only happen once every 7 days =cut our $EXIM_LOG_DIR = '/var/log'; our $PID_FILE = '/var/run/slurp_exim_mainlog.pid'; our $OUTPUT_LOG_DIR = '/var/cpanel/logs'; our $OUTPUT_LOG_FILE = 'eximstats_sqlite_import.log'; our $MAX_OUTPUT_LOG_AGE = 30 * 24 * 60**2; # 30 days our $TIME_BETWEEN_IMPORTS = 7 * 24 * 60**2; # 7 days sub _OPTIONS { return qw( force reimport ); } __PACKAGE__->new(@ARGV)->script() unless caller(); ## CAVEATS re: case 53744 ## 1. the eximstats schema handles dedupes, so we process all recent files ## 2. TaskQueue is overkill for this one-time slurp sub script { my ($self) = @_; my $time = _time(); ## skip the slurp of exim_mainlog* if the touch file exists and it hasn't been TIME_BETWEEN_IMPORTS yet if ( !$self->getopt('force') ) { if ( $self->getopt('reimport') ) { my $last_import_time = ( stat $Cpanel::EximStats::ImportInProgress::IMPORTED_FILE )[9] || 0; my $seconds_since_last_import = ( $time - $last_import_time ); if ( -e $Cpanel::EximStats::ImportInProgress::IMPORTED_FILE && $TIME_BETWEEN_IMPORTS > $seconds_since_last_import ) { print "[slurp_exim_mainlog] Skipping re-import because the logs were last imported at " . scalar localtime($last_import_time) . ", and the system only allows imports every $TIME_BETWEEN_IMPORTS seconds without the --force flag.\n"; return; } } else { if ( -e $Cpanel::EximStats::ImportInProgress::IMPORTED_FILE ) { print "[slurp_exim_mainlog] Skipping import because the logs were already imported.\n"; return; } } } my $IMPORT_LOG_AGE_LIMIT = 60**2 * 24 * ( int( Cpanel::EximStats::Retention::get_valid_exim_retention_days() ) || 60 ); Cpanel::PIDFile->do( $PID_FILE, sub { my %imports; for my $log ( grep { m/^exim_mainlog/ } @{ Cpanel::FileUtils::Dir::get_directory_nodes($EXIM_LOG_DIR) } ) { # Skip archived logs for this import next if substr( $log, -3 ) eq '.gz'; my $age = ( stat("$EXIM_LOG_DIR/$log") )[9]; ## eximstats shows info for 90 days by default, but it's configurable next if ( $time - $age > $IMPORT_LOG_AGE_LIMIT ); $imports{"$EXIM_LOG_DIR/$log"} = $age; } my @imports = keys %imports ? map { $_->[0] } sort { $b->[1] <=> $a->[1] } map { [ $_, $imports{$_} ] } keys %imports : (); Cpanel::FileUtils::TouchFile::touchfile($Cpanel::EximStats::ImportInProgress::IMPORTED_FILE); $self->call_import_exim_data( \@imports ); } ); return; } # In a function for tests sub call_import_exim_data { my ( $self, $imports_ar ) = @_; if ( !scalar @$imports_ar ) { print "[slurp_exim_mainlog] there were no exim_mainlog files new enough to import.\n"; return; } $self->remove_old_output_logs(); my $log_file = "$OUTPUT_LOG_DIR/$OUTPUT_LOG_FILE." . Cpanel::Time::ISO::unix2iso(); print "[slurp_exim_mainlog] starting import of the exim_mainlog files. Output will be logged to: $log_file\n"; Cpanel::Daemonizer::Tiny::run_as_daemon( sub { close(STDIN); open( STDIN, "<", "/dev/null" ); open( my $log_fh, '>>', $log_file ); open( STDOUT, ">&=", $log_fh ); open( STDERR, ">&=", $log_fh ); exec( '/usr/local/cpanel/scripts/import_exim_data', @$imports_ar ); } ); return; } sub remove_old_output_logs { my ($self) = @_; my @logs_to_remove; for my $log ( grep { m/^\Q$OUTPUT_LOG_FILE\E\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$/ } @{ Cpanel::FileUtils::Dir::get_directory_nodes($OUTPUT_LOG_DIR) } ) { my $log_path = "$OUTPUT_LOG_DIR/$log"; next if !-e $log_path || _time() - ( stat(_) )[9] < $MAX_OUTPUT_LOG_AGE; push @logs_to_remove, $log_path; } try { Cpanel::Autodie::Unlink::unlink_if_exists_batch(@logs_to_remove) if scalar @logs_to_remove; } catch { warn "[slurp_exim_mainlog] there was a problem removing old output logs: " . Cpanel::Exception::get_string_no_id($_); }; return; } # For tests sub _time { return time(); }