pipeline status

Tools to submit UWS jobs with CLI

Installation

In a directory of your choice, clone this version of uws-client.

git clone https://github.com/aicardi-obspm/uws-client
cd uws-client
git checkout python3-support
pip install .

This installs uws into your systems bin directory and makes the command available on the command line.

Shell sample

The file scripts/job.sh contains a sample shell script to launch a job, wait for its completion et retrieve its results.

#!/bin/bash

## Configure the following variables
SERVER="myserver.mydomain"
JID="jobname"
LOGIN="login"
TOKEN="token"


FILE="$1"

TMPDIR=`mktemp -d`
uws -H https://$SERVER/rest/$JID -U $LOGIN --password $TOKEN job new --run config=@$FILE runId=test_job_uws > $TMPDIR/uws_submission

if grep -q 'An error occurred:' $TMPDIR/uws_submission ; then
	echo "error with job submission"
	cat $TMPDIR/uws_submission
	rm -rf $TMPDIR
	exit 2
fi


JOBID=` grep '^Job ID:' $TMPDIR/uws_submission | cut -f3 -d\ `

echo "Job: $JOBID"

while true; do
	STATUS=`uws -H https://$SERVER/rest/$JID -U $LOGIN --password $TOKEN job phase $JOBID`
	if echo "$STATUS" | grep -q "An error" ; then
		echo "Error with uws"
		echo "$STATUS"
		rm -rf $TMPDIR
		exit 2
	fi
	if [ "$STATUS" == "COMPLETED" ]; then
		echo "Job completed"
		uws -H https://$SERVER/rest/$JID -U $LOGIN --password $TOKEN job results $JOBID
		rm -rf $TMPDIR
		exit 0
	elif [ "$STATUS" == "ERROR" ]; then
		echo "Job completed with error"
		rm -rf $TMPDIR
		exit 1
	fi
	sleep 2
done

Pure REST shell sample

If you don’t want to install uws-client, the file scripts/job-rest.sh contains a sample shell scripts to lauch a job using only cURL and basic shell commands.

#!/bin/bash

## Configure the following variables
SERVER="myserver.mydomain"
JID="jobname"
LOGIN="login"
TOKEN="token"


## Parameter file is passed as first argument
FILE="$1"

TMPDIR=`mktemp -d`

# Creates a job and get its id
jobid=`curl -i -s -X POST --basic -u "$LOGIN:$TOKEN" -F config=@$FILE -F runId=test_job https://$SERVER/rest/$JID/ | grep '^Location' | sed -e 's,.*/,,;s/\r//'`

# Start the job
curl -X POST -s --basic -u "$LOGIN:$TOKEN" -F PHASE=RUN https://$SERVER/rest/$JID/$jobid/phase/

echo "Job: $jobid"

while true; do
	# fetch the job state (which is an xml file) and the uws:phase value
	STATUS=`curl -s --basic -u "$LOGIN:$TOKEN" https://$SERVER/rest/$JID/$jobid/ | sed -ze 's/.*<uws:phase>//;s,</uws:phase>.*$,,'`
	if [ "$STATUS" == "COMPLETED" ]; then
		echo 
		echo "Job completed"
		echo "Results :"
		# fetch the list of results of the job and loop over each uws:result entity
		curl -s --basic -u "$LOGIN:$TOKEN" https://$SERVER/rest/$JID/$jobid/results/ |sed -e 's/>/>\n/g' | grep '<uws:result ' > $TMPDIR/results.xml
		while read -r line ; do 
			# a result looks like <uws:result id="filename" mime-type="something" xlink:href="url" />
			filename=`echo $line | sed -e 's/.*id="//;s/".*$//'`
			url=`echo $line | sed -e 's/.*xlink:href="//;s/".*$//'`
			echo "Get $filename"
			curl -s --basic -u "$LOGIN:$TOKEN" $url -o $filename
		done < $TMPDIR/results.xml
		rm -rf $TMPDIR

		exit 0
	elif [ "$STATUS" == "ERROR" ]; then
		echo 
		echo "Job completed with errors :"
		curl -s --basic -u "$LOGIN:$TOKEN" https://$SERVER/rest/$JID/$jobid/stderr
		exit 1
	fi
	echo -n "."
	sleep 2
done

exit 0

Python sample

The file scripts/job.py contains a sample python script to launch a job, wait for its completion et retrieve its results.

#!/usr/bin/env python3

from uws import UWS

## Configure the following variables
SERVER="myserver.mydomain"
JID="jobname"
LOGIN="login"
TOKEN="token"

## Add any parameter to your job here
parameters = {'config':'@myconfigurationfile',
              'runId':'my_job_name'}

## 
uws_client = UWS.client.Client(url=f"https://{SERVER}/rest/{JID}", user=LOGIN, password=TOKEN)


job = uws_client.new_job(parameters)
job = uws_client.run_job(job.job_id)

print(f"Job : {job.job_id}")

while True:
    time.sleep(2)
    phase = uws_client.get_phase(job.job_id)
    if phase == UWS.models.JobPhases.COMPLETED:
        print("Job completed")
        break
    elif phase == UWS.models.JobPhases.ERROR or phase == UWS.models.JobPhases.ABORTED:
        print("Job failed")
        break

job = uws_client.get_job(job.job_id)
for result in job.results:
    filename = "./" + result.rid
    print(f"Downloading {result.rid}")
    uws_client.connection.download_file(str(result.reference), LOGIN, TOKEN, filename)

PHP REST sample

The file scripts/job-rest.php contains a sample PHP script using the REST interface.

<?php 
## Configure the following variables
$SERVER="myserver.mydomain";
$JID="jobname";
$LOGIN="login";
$TOKEN="token";

## Parameter file is passed as first argument
$FILE=$argv[1];

function post_uws($parameter, $path="") {
  global $SERVER,$JID,$LOGIN,$TOKEN;
  $curl = curl_init();
  curl_setopt($curl, CURLOPT_URL, "https://$SERVER/rest/$JID/".$path);
  curl_setopt($curl, CURLOPT_POST, true);
  curl_setopt($curl, CURLOPT_POSTFIELDS, $parameter);
  curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
  curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
  curl_setopt($curl, CURLOPT_USERPWD, $LOGIN . ":" . $TOKEN);

  $return = curl_exec($curl);
  curl_close($curl);
  return $return;
}
function get_uws($path="") {
  global $SERVER,$JID,$LOGIN,$TOKEN;
  $curl = curl_init();
  curl_setopt($curl, CURLOPT_URL, "https://$SERVER/rest/$JID/".$path);
  curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
  curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
  curl_setopt($curl, CURLOPT_USERPWD, $LOGIN . ":" . $TOKEN);

  $return = curl_exec($curl);
  curl_close($curl);
  return $return;
}
function get_result($url) {
  global $SERVER,$JID,$LOGIN,$TOKEN;
  $curl = curl_init();
  curl_setopt($curl, CURLOPT_URL, $url);
  curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
  curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
  curl_setopt($curl, CURLOPT_USERPWD, $LOGIN . ":" . $TOKEN);

  $return = curl_exec($curl);
  curl_close($curl);
  return $return;
}
# Creates a job and get its id
$cfile = curl_file_create($FILE, 'text/plain', $FILE);
$parameter = array('config' => $cfile);


$xml = new SimpleXMLElement(post_uws($parameter));
$items = $xml->xpath('./uws:jobId');
$job_id = $items[0];

post_uws(array('PHASE'=>'RUN'), "$job_id/phase/");

print("$job_id\n");

while (true) {
  $xml = new SimpleXMLElement(get_uws("$job_id/"));
  $items = $xml->xpath('./uws:phase');
  $status = $items[0];
  if ($status == "COMPLETED") {
	  print("\nJob completed\n");
	  $xml = new SimpleXMLElement(get_uws("$job_id/results"));
	  $items = $xml->xpath('./uws:result');
	  foreach($items as $item) {
             $filename = $item->attributes()['id'];
	     $url = $item->attributes('http://www.w3.org/1999/xlink')[0];
	     print("get $filename from $url\n");
	     file_put_contents($filename, get_result($url));
	  }

	  break;
  }
  elseif ($status == "ERROR") {
	  print("\nJob error :\n");
	  print(get_uws("$job_id/stderr"));
	  print("\n");
	  break;
  }
  print(".");
  sleep(2);
}
exit(0);

?>