#!/usr/bin/perl

## RedHotIngest.pl is a script that runs through the hot-folder mechanism of Zoom,
## but pre-empts ingest through that mechanism by invoking zm import from within.

use XML::Simple;

## Hard-coded arguments - modify according to setup
my $SERVER_URL = "http://dam1.convergencia.local:8880";
my $HF_USER = "hotfolder";
my $HF_PASSWORD = "hotfolder1";
my $HOT_FOLDER = "/MediaGrid/zoom/HotFolders/VideoArchivo";
my $DELIMITER = "\t";
my $CSV = ""; # change this to "" (empty) if you change DELIMITER to "\t"
my $REPO_DEST =  "/Television"; # Project name is: "/Video Archivo"
my $ZM = "/home/zoom/zoom/bin/zm"; # give FQ path to zm
my $METAFILE = "metafile.txt";
my $LOGFILE = "/MediaGrid/zoom/Scripts/pre-ingest.log";
my $LOGFILE_ARG = "\"/MediaGrid/zoom/Scripts/pre-ingest.log\"";
my $INACTIVE_THRESHOLD = 120; ## make sure to keep this value <= HF inactive interval - else the files left over by this script as being ineligible for import, will get imported by the HF ingest -- without the relevant MD!

## Assumptions
# Will skip importing any xml file that may be present in the hot folder
# Will specifically look for CLIP xml tag, and parse only the metadata xml containing that; other xml files will be ignored
# Will only import files flatly to the repo, so hierarchies inside hot folder will get lost
# zm executable is in path  (or edit $ZM to point to fq path of zm exe)
# All newlines and tabs within the property values are replaced by ";" (We could not use CSV because there are commas also within; we had to switch to tab)!
# Hot folder, Log file folder must already exist
# Two different files with same names but different extensions (sample.mov, sample.mp4) will not be able to reference 2 different xml files - both will end up looking for sample.xml)
###

## Actual script - do not modify for parameter changes
## MAIN

# List all files in the hot folder location
opendir(D, "$HOT_FOLDER") || die "Can't open directory $d: $!\n";
my @list = readdir(D);
closedir(D);

open my $logHandle, '>>', $LOGFILE or die "...$!";
print $logHandle "\n\n".localtime()."\n";

chdir $HOT_FOLDER;
open my $mdh, '>:encoding(UTF-8)', $METAFILE or die "...$!";

my @infiles;
my @xmlMetaFiles;
my @propNames;
my $index = 0;
my $xmlLevel = 0;

# for each file do the following:
foreach my $f (@list) {
  if($f eq $METAFILE) {
    next;
  }
  parseMetadata($f);
}
close $mdh;

$infilesCount = @infiles;
print $infilesCount."\n";
if($infilesCount == 0) {
  # metafile created by this pre-ingest script, so clean it up anyway on exit!
  print $logHandle "No eligible files to import\n";
  unlink $METAFILE or print $logHandle "Can't delete $METAFILE\n";
  close $logHandle;
  exit;
}

# Now take these infiles, and do a zm import with the constructed metadata file
my @importArgs = ($ZM, "-s", $SERVER_URL, "--username", $HF_USER, "--password", $HF_PASSWORD, "--thisCommandOnly", "im", "-m", "import-through-pre-ingest-script", "-d", "\"$REPO_DEST\"", $csv, "-mf", $METAFILE);
# add infiles to the args list here -
foreach my $i (@infiles) {
  push (@importArgs, "\"$i\"");
}

push (@importArgs, ">>$LOGFILE_ARG 2>&1");
my $log = join(' ', @importArgs);
$log .= "\n";
print $log;
print $logHandle $log;

# add stderr redirect
@result = `@importArgs`;
$status = ($? >> 8);
print $logHandle "Import status: $status\n";
if($status == 0) {
  # Import successful -
  # delete infiles
  foreach (@infiles) {
    unlink $_ or print $logHandle "Can't delete $_\n";;
  }

  # delete metafils
  foreach (@xmlMetaFiles) {
    unlink $_ or print $logHandle "Can't delete $_\n";;
  }
}
# metafile created by this pre-ingest script, so clean it up anyway on exit!
unlink $METAFILE or print $logHandle "Can't delete $METAFILE\n";
close $logHandle;

############### SUBWAY AHEAD ##############

sub parseMetadata {
  my $f = shift;
  if($f =~ /\.xml$/i) { # this is an xml file, not for ingest but present here as md source for some other file!!
    return;
  }
  if($f =~ /^\.$/ || $f =~ /^\.\.$/) {
    return;
  }

  $mTime = (stat($f))[9];
  $now = time();
  print "inactive = ".($now - $mTime)."\n";

  if(($now - $mTime) < $INACTIVE_THRESHOLD) {   # do not process files that have been touched within the last x seconds...
    print $logHandle "Too hot: $f\n";
    return;
  }

  push (@infiles, $f);
  my $mdF = $f;

  $mdF =~ s/\..*$/\.xml/i;
  if(-e "$mdF") {
    # there is a .xml corresponding to the input file; must parse this!
        push (@xmlMetaFiles, $mdF);
    extractMetadata($mdF);
  }
}

sub extractMetadata { # input: metadata xml file name
  $md = shift;
  my $dataRef = XMLin($md, NoAttr=>1);
  $xmlLevel = 0;
  parse($dataRef);
  $index++;
}

sub parse { # input: reference to hash
  my $ref = shift;
  my %data = %{$ref};
  my %flat;

  foreach my $k (keys %data) {
    my $value = $data{$k};
    if(ref($value) eq 'HASH') {
      local $valid = validateXml($k);
          if($valid == 0) {
            return;
          }
          parse($value);
          return;
        } else {
          $value =~ s/[\t]+/;/g;
          $value =~ s/[\n]+/;/g;
          $flat{$k} = $value;
        }
  }

  if($index == 0) {
    foreach my $k (keys %flat) {
          push(@propNames, $k);
      print $mdh "$k$DELIMITER";
    }
        print $mdh "\n";
  }

  foreach my $k (@propNames) {
    print $mdh "$flat{$k}$DELIMITER";
  }
  print $mdh "\n";
  $xmlLevel++;
}

sub validateXml {       # hard-coded to match CLIPS->CLIP index 0
  $k = shift;
  if($xmlLevel == 0) {
        # Check if first hash is "CLIP"; if not, do not proceed with metadata parsing!
        #print $k."\n";
        if($k ne 'CLIP') {
          return 0;
        }
    return 1;
  }
}
