#!/usr/bin/perl

######################################################################
##
## repa_log
##
## Created 15.4.1998 by Lasse A Vartiainen
##
## $Modified: 2015- 2-03 10:23:55 by lav@iki.fi $
##
## Tries to fix logs made by bugified tintin1.64dev (Ie. removing
## duplicates strings in a line).
##
######################################################################

#--- Initialize ---
use warnings;
use strict;

use Getopt::Long;
use English; # Allows the use of some more legible system variables

use FindBin;
use lib "$FindBin::Bin";

use Utils qw(verbose debug debugf fatal alert print_r htime bail);
use File::Slurp qw(read_file write_file);

#--- Begin ---
our $REM_ONLY_DUPLICATE = 0;

our $REM_TIME           = 1;
our $REM_ZERO           = 0;
our $REM_DUPL           = 1;
our $REM_ANSI           = 0;

our $TRASH              = "/files/alogs/trash";

our $DEMO               = 0;
our $FORCE              = 0;
our $VERBOSE            = 1;

$| = 1 if $VERBOSE;       #Nonbuffered output if verbose

Getopt::Long::config("auto_abbrev","ignore_case","getopt_compat","permute","norequire_order",
                     "nobundling","nopass_through");
GetOptions
    (
     "time!"           => \$REM_TIME,
     "duplicates!"     => \$REM_DUPL,
     "zeroes!"         => \$REM_ZERO,

     "demo!"           => \$DEMO,
     "only_duplicates" => \$REM_ONLY_DUPLICATE,
     "ansi!"           => \$REM_ANSI,
     "verbose:i"       => \$VERBOSE,
     "force!"          => \$FORCE,
     );

select(STDOUT); $|=1;

Utils::verbose($VERBOSE);
Utils::logfile("/dev/shm/repa_logs.log");

# *** MAIN PART
#printf("%d - %d - %d - %d\n",$REM_ZERO,$REM_DUPL,$REM_ANSI,$REM_TIME);

if($REM_ONLY_DUPLICATE) {
    remove_only_duplicates(); exit 0;
}

my @paths = ();
foreach my $try (@ARGV) {
    if(-f $try) {
        push(@paths, $try);
    } elsif(-d $try) {
        opendir(DIR, $try) || die;
        push(@paths, sort map { "$try/$_" } grep { m/nil_/ } readdir(DIR));
        close(DIR);
    } elsif(not -e $try) {
        debugf(1, "  - WARN: Ignoring given non-existing logfile : %s\n", $try);
    } else {
        fatal("ERROR: Argument neither file nor directory : <$try>\n");
    }
}

my $files = scalar(@paths);
my $START = htime();
my $total_lines = 0;
foreach my $file (@paths) {
    my $data      = File::Slurp::read_file($file, array_ref=>1) or fatal("ERROR: Failed slurping <$file> : $!\n");
    my $len_orig  = length(join("", @$data));
    my $max       = scalar(@$data);
    $total_lines += $max;
    debugf(2, "* %5d : %5d lines in %s\n", $files--, $max, $file);

    remove_time($data, $max) if($REM_TIME);
    # remove_zero($data, $max) if($REM_ZERO);
    remove_dupl($data, $max) if($REM_DUPL);
    remove_ansi($data, $max) if($REM_ANSI);


    my $len_new  = length(join("", @$data));
    if(!$FORCE and $len_new <= 0.5*$len_orig) {
        alert(sprintf("Size shrink to less than half (%2.0f%%), use '--force' to allow\n", 100*$len_new/$len_orig));
    } elsif(!$DEMO) {
        my $mtime = (stat($file))[9] or fatal("Failed to get mtime for <$file> : $!\n");
        rename($file, "$TRASH/$file") if(-d $TRASH);
        # debug(3, "Backuping <$file> to <$file\~>...\n");
        # qx(cp -fp \"$file\" \"$file\~\");

        debug(3, "- Writing results, backup copied to $TRASH/$file\n");
        write_file($file, $data) or fatal("Failed to %dkB to <%s> : %s\n", $len_orig/1E3, $file, $!);
        utime(time(), $mtime, $file) or fatal("Failed utime($mtime, $file) : $!\n");

    } else {
        debugf(3, "- DEMO: Skipping writing\n");
    }
}
debugf(2, "- Processed %d files in %.0fs, %.0fk lines/s\n", scalar(@paths), htime()-$START, ($total_lines/1E3) / (htime()-$START) );
debug(1, "Finished.\n");

exit 0;

sub remove_time {
    my($lines, $max) = @_;

    my $fixed = 0;
    my $start = htime();
    debug(2, "- Scanning time_stamps : ");

    foreach my $i (grep { index($lines->[$_], '') != -1 } (0..$max-1)) {
        $fixed++ if($lines->[$i] =~ s|\d+||);
    }
    debugf(2, "%d removed, %.0fk lines/s\n", $fixed, ($max/1E3) / (htime()-$start) );
    # print @$lines;
}

# Ancient obsolote fix, for something long forgotten
sub remove_zero {
    my($lines, $max) = @_;

    my $fixed = 0;
    my $start = htime();
    debug(2, "- Scanning zeroes : ");

    foreach my $i (grep { index($lines->[$_], "\000") } (0..$max-1)) {
        $fixed++ if($lines->[$i] =~ s|\000[^n]{0,2}||g);
    }
    debugf(2, "%d removed, %.0fk lines/s\n", $fixed, ($max/1E3) / (htime()-$start) );
    print @$lines;
}

sub remove_dupl {
    my($lines, $max) = @_;
    debug(2, "- Scanning duplicates  : ");
    my $start         = htime();
    my $count_unmatch = 0;
    my $min           = 4;

    my $count = 0;

    foreach my $i (grep { index($lines->[$_], substr($lines->[$_], 0, $min), $min) != -1 } 0..$max-1) {
        # debugf(3, "%02d%% ", 100*$i/$max) if($VERBOSE and $i%1000==0);
        next if(substr($lines->[$i],0,10) eq "----------");

        #Skip lines consisting of single repeated char
        if($lines->[$i] =~ m/\A(.)\1*\Z/) {
            debugf(3, "\n- Unmatch : %s", $lines->[$i]);
            $count_unmatch++;

        } elsif($lines->[$i] =~ s/\A([^\n]{4,})(\1)(.*\n)\Z/$2$3/) {
            my $rem = $1;
            if($rem =~ m/\A(.)\1*\Z/) {
                #print STDERR "RESTORED: <$rem>\n";
                $lines->[$i] = $rem.$lines->[$i];
            } else {
                $count++;
            }
        }
        # elsif($lines->[$i] =~  m/\A([^\n]{3,})(\1)(.*\n)\Z/) {
        #    print STDERR "SKIP: $lines->[$i]"; #XXX
        #}
    }
    debugf(2, "%d lines fixed, ignored %d, %.0fk lines/s \n", $count, $count_unmatch, ($max/1E3) / (htime()-$start));
}



#Remove duplicates straigth from a file, printing result to stdout
#Faster than holding the file at memory
sub remove_only_duplicates {
    debug(1, "Removing duplicate lines...\n");
    my($count_corr, $count_unmatch)=(0,0);
    my($lines)=0;
    while(<>) {
        my $line = $_;
        if($VERBOSE and ($lines++)%4096 == 0) { debug(3, "$lines.."); }

        #Skip lines repeating same char
        if($line =~ m/^(.)\1{4,}$/) {
            $count_unmatch++;
        } elsif($line =~  m/^([^\n]{4,})(\1)(.*\n)/o) {
            $line = "$2$3";
            $count_corr++;
        }
        print $line;
    }
    debug(2, "\n\tcorrected $count_corr lines (detected $count_unmatch single_char lines)\n");
}

sub remove_ansi {
    my($lines, $max) = @_;

    my $fixed = 0;
    my $start = htime();
    debug(3, "- Scanning ansi_codes : ");

    foreach my $i (grep { index($lines->[$_], "\e") != -1 } (0..$max-1)) {
        $fixed++ while($lines->[$i] =~ s/\e\[[0-9m\;]*m//g);
        $fixed++ while($lines->[$i] =~ s/\e[\[0]*//g);
    }
    debugf(3, "- %d lines fixed, %.0fk lines/s\n", $fixed, ($max/1E3) / (htime()-$start) );
}


#--- Functions ---
sub save_file {
    my($filename,$data)=@_;
    my($tmp);

    open (SFILE,">$filename")|| return -2;
    $tmp = syswrite(SFILE,$data,length($data));
    if($tmp != length($data)) {
        error("Wrote $tmp of ".length($data)." bytes to $filename",0);
        return -3;
    }
    close(SFILE);
    return 0;
}


## Local Variables:
## mode: perl
## make-backup-files: t
## End:
