Commit 3ddf9769 authored by Karl Kornel's avatar Karl Kornel
Browse files

Switch from drive to gdrive. 

drive_run now does two executions: First we do a search for the
specified library ID.  We then grab the file unique ID, and download it
directly to the SampleSheet.txt file.

Since we are pulling files from Google Drive directly into the
runfolder, we don’t need to do all of the initial pull and directory-
maintenance stuff that we had to do before.  We also don’t have to deal
with things like clashes.  In all, it’s much cleaner!
parent db78a1ee

bbox-workflow-nextseq

+123 −107
Original line number Diff line number Diff line
@@ -48,11 +48,8 @@ my @RUNCOMMAND = (qw( 
/usr/local/bin/bcl2fastq -l WARNING -o Data/Intensities/BaseCalls -p 9 -d 2 --no-lane-splitting

));



# $DRIVECOMMAND is the path to the drive command

my $DRIVECOMMAND = '/home/bufferbox/gopath/bin/drive';



# $DRIVEPATH is the path to the drive folder containing samplesheets

my $DRIVEPATH = '/home/bufferbox/gdrive/NS Samplesheets';

# $DRIVECOMMAND is the path to the gdrive command

my $DRIVECOMMAND = '/home/bufferbox/gopath/bin/gdrive';



# Next, let's make a big list of global variables.  These all initially start

# out as undefined, and are filled in as we go along.
@@ -407,102 +404,30 @@ EOF 
		return;

	}



	# Go into the drive folder

	dolog("Checking gdrive folder\n");

	chdir($DRIVEPATH) or do {

		dolog(<<"EOF");

Unable to access the Google Drive folder containing the samplesheets.

The folder we are trying to access is: $DRIVEPATH

EOF

        workflow_complete($RUNFOLDER);

		email_failure("Could not access gdrive folder", 'run',		                              $RUNFOLDER, "$RealBin/email-recipients.txt",

			      $LOGPATH, \$LOG, $RealBin, $RealScript);

		return;

	};



	# Run our first drive pull

	my $drive_exit_code;

	dolog("Running drive pull\n");

	$drive_exit_code = drive_run($DRIVECOMMAND, 'pull', '-no-prompt',

		                     '-ignore-conflict', '-verbose', '.');

	return unless defined($drive_exit_code);



	# If we got return code 13, then we need to fix clashes and re-pull

	while ($drive_exit_code == 13) {

		$drive_exit_code = drive_run($DRIVECOMMAND, 'pull',

                                     '-no-prompt', '-verbose',

                                     '-ignore-conflict', '-fix-clashes');

		return unless defined($drive_exit_code);

		$drive_exit_code = drive_run($DRIVECOMMAND, 'pull', '-ignore-conflict',

                                     '-no-prompt', '-verbose', '.');

		return unless defined($drive_exit_code);

	}



    # If we got any other non-zero exit code, we have a problem.

    if ($drive_exit_code != 0) {

        dolog("That's bad 8-(\n");

        workflow_complete($RUNFOLDER);

        email_failure('Un-good exit code from the drive program', 'run',

                      $RUNFOLDER, "$RealBin/email-recipients.txt",

                      $LOGPATH, \$LOG, $RealBin, $RealScript);

        return;

    }



	# Find our samplesheet

	dolog("Drive running complete!  Checking for samplesheet.\n");

	dolog("Information extraction complete!  Checking for samplesheet.\n");



    # If we have a SampleSheet.csv already, then we're fine

    # If we have a SampleSheet.csv already, then we're fine.

    # Otherwise, we'll need to download the samplesheet from Google Drive.

    if (-r "$RUNFOLDER/SampleSheet.csv") {

        dolog("A SampleSheet.csv already exists.  Not copying from gdrive\n");

    } else {

        # Build the list of candidate SampleSheet files.

    	my $samplesheet_search_path = "$DRIVEPATH/$LIBRARYID SampleSheet*.csv";

    	$samplesheet_search_path =~ s{[ ]}{\\\ }g;

    	my @samplesheets = glob $samplesheet_search_path;

    	dolog('Found ' . scalar(@samplesheets) . " candidate(s).\n");



        # Pick exactly which samplesheet to use (if any were found).

    	my $samplesheet_path;

	    if (scalar(@samplesheets) == 0) {

		    dolog("No samplesheets found!\n");

            workflow_complete($RUNFOLDER);

	    	email_failure("No samplesheets found for $LIBRARYID", 'run',

	                      $RUNFOLDER, "$RealBin/email-recipients.txt",

	                      $LOGPATH, \$LOG, $RealBin, $RealScript);

        dolog("Downloading samplesheet for ${LIBRARYID} from Google Drive.\n");

        $drive_exit_code = drive_run($DRIVECOMMAND, $LIBRARYID,

                                "$RUNFOLDER/SampleSheet.csv");



        # Make sure drive exited OK

        if (!defined($drive_exit_code)) {

            return;

	    } elsif (scalar(@samplesheets) == 1) {

    		$samplesheet_path = $samplesheets[0];

	    } else {

		    # Get the last-modified time for each samplesheet

    		my %samplesheet_dates;

	    	foreach my $samplesheet (@samplesheets) {

		    	my $mtime = (stat($samplesheet))[9];

			    dolog("Samplesheet $samplesheet last modified $mtime\n.");

    			$samplesheet_dates{$samplesheet} = $mtime;

        }



		    # Sort the samplesheets, and get the newest

    		my @sorted_samplesheets = sort {

	    		$samplesheet_dates{$a} <=> $samplesheet_dates{$b};

		    } keys(%samplesheet_dates);

    		$samplesheet_path = pop @sorted_samplesheets;

        elsif ($drive_exit_code != 0) {

            dolog("The drive command exited with code ${drive_exit_code}.\n");

            dolog("That's bad 8-(\n");

            return;

        }

        else {

            dolog("The drive command exited cleanly!\n");

        }

    	dolog("Using samplesheet at path $samplesheet_path\n");



        # Copy the SampleSheet into place

    	File::Copy::copy($samplesheet_path, "$RUNFOLDER/SampleSheet.csv") or do {

    		dolog(<<"EOF");

We were unable to copy the SampleSheet, because of an error.

The file we tried to copy is: $samplesheet_path

The destination was: $RUNFOLDER/SampleSheet.csv

The error is: $!

EOF

            workflow_complete($RUNFOLDER);

    		email_failure('Unable to copy SampleSheet.csv into place', 'run',

                          $RUNFOLDER, "$RealBin/email-recipients.txt",

                          $LOGPATH, \$LOG, $RealBin, $RealScript);

    	};

    	dolog("Samplesheet copied\n");

    }



	# Check to see if we have any Project directories
@@ -729,23 +654,113 @@ sub dolog { 
	return log_msg($LOGHANDLE, \$LOG, $msg);

}



# drive_run: Run the drive command, and return.

# drive_run: Run the drive command, write output and return.

# Returns the exit code, or undef on failure.

sub drive_run {

	my @drive_options = @_;

    my ($command_path, $library_id, $output_path) = @_;



    # Open our output file

    # NOTE: We DO overwrite files here, so it's up to the caller to

    # ensure that the file is OK to override.

    my $output_handle;

    open($output_handle, '>', $output_path) or do {

        dolog(<<"EOF");

There was a problem opening the samplesheet file for writing!

The file we tried to write to: $output_path

The error we got: $!

EOF

        email_failure('Problem writing to samplesheet.csv', 'run',

                  $RUNFOLDER, "$RealBin/email-recipients.txt",

                  $LOGPATH, \$LOG, $RealBin, $RealScript);

        return undef;

    };



    # Work out our drive search command:

    # * Make the most recent item appear first on the list.

    # * Only return one item (so, we get the most recent).

    # * Limit to files containing "XXXX samplesheet" (case-insensitive)

    # * Limit to files containing ".csv"

    my @drive_options = (

        $command_path,

        'list',

        '--order',

        'recency desc',

        '-m',

        '1',

        "(name contains '${library_id} samplesheet') AND (name contains '.csv')",

    );



	# Output some header stuff

	dolog('Drive command: ' . join(' ', @drive_options) . "\n");

	dolog("Drive command output:\n");

	dolog('Drive search command: ' . join(' ', @drive_options) . "\n");

	dolog("Drive search command output:\n");

	dolog("==========================================================\n");



	# Actually spawn the command

    # Actually spawn the search command.

    my $search_inhandle;

    my $results_handle = gensym();

    my $search_buffer = '';

    my $search_pid;

    eval {

        $search_pid = open3($search_inhande, $results_handle, $results_handle,

                        @drive_options);

    };

    if ($@) {

        dolog(<<"EOF");

There was a problem running the drive command to search for files.

The error we got was: $@

EOF

		email_failure('Problem runnig the drive command', 'run',

			      $RUNFOLDER, "$RealBin/email-recipients.txt",

			      $LOGPATH, \$LOG, $RealBin, $RealScript);

    }

    close($search_inhandle);



	# Pull output into the buffer, and then clean up.

	while (read($results_handle, $drive_buffer, 10)) {

        1;

	}

    dolog($search_buffer);

	waitpid($search_pid, 0);

	my $drive_search_exit_code = $? >> 8;

	close($results_handle);



    # Get the file UID with a fun compound split:

    # First, split the search buffer into multiple lines.

    # Grab the second line.

    # Next, split the retrieved line into space-separated fields.

    # The first field is our UID.

    my $file_uid = (split(/\s+/, (split(/\n/, $search_buffer))[1]))[0];

    if (!defined($file_uid) or !$file_uid) {

        dolog(<<"EOF");

We could not find a matching sample sheet for this library ID.

The library ID is: $library_id

Please make sure the samplesheet name starts with "$library_id SampleSheet",

and that the name ends in '.csv'.

EOF

		email_failure('Unable to find matching samplesheet', 'run',

			      $RUNFOLDER, "$RealBin/email-recipients.txt",

			      $LOGPATH, \$LOG, $RealBin, $RealScript);

        return undef;

    }



    # NOW we can finally get our file!

    my @drive_options = (

        $command_path,

        'download',

        '--stdout',

        $file_uid,

    );



	# Spawn the downlaod command

    # Remember, $output_handle was opened up a while ago.

    # We'll let IPC::open3 take care of writing to the file directly.

	my $input_handle;

	my $output_handle = gensym();

    my $output_id = '&>' . fileno($output_handle);

    my $error_handle = gensym();

	my $drive_buffer = '';

	my $drive_pid;

	eval {

		$drive_pid = open3($input_handle, $output_handle, $output_handle,

		$drive_pid = open3($input_handle, $output_id, $error_handle,

                        @drive_options);

	};

	if ($@) {
@@ -760,12 +775,13 @@ EOF 
	}

	close($input_handle);



	# Send any output to the log, and then clean up

	while (read($output_handle, $drive_buffer, 10)) {

	# Send any stderr to the log, and then clean up

	while (read($error_handle, $drive_buffer, 10)) {

		dolog($drive_buffer);

		$drive_buffer = '';

	}

	waitpid($drive_pid, 0);

	close($error_handle);

    close($output_handle);



	# Get and return the exit code