changeset 0:92ca20fdb790 draft default tip

planemo upload for repository https://github.com/galaxyecology/tools-ecology/tree/master/tools/ogc_api_processes_wrapper commit 8b4b58222af2c77abd41dd8f17862a24ca7d3381
author ecology
date Fri, 06 Sep 2024 10:30:22 +0000
parents
children
files crim_project_ogc_api_processes.xml macros.xml ogc_api_processes_wrapper.R test-data/crim_test_input.json test-data/product_url.txt
diffstat 5 files changed, 493 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/crim_project_ogc_api_processes.xml	Fri Sep 06 10:30:22 2024 +0000
@@ -0,0 +1,121 @@
+<tool id="hirondelle_crim" name="Hirondelle CRIM OGC API Processes" version="0.2.0" profile="22.05">
+	<description>executes remote processes.</description>
+	<macros>
+		<import>macros.xml</import>
+	</macros>
+	<expand macro="requirements"/>
+	<command detect_errors="exit_code"><![CDATA[
+	Rscript '$__tool_directory__/ogc_api_processes_wrapper.R'
+		--outputData '$output_data'
+]]></command>
+	<configfiles>
+		<inputs name="inputs" filename="inputs.json" data_style="paths"/>
+	</configfiles>
+	<inputs>
+		<conditional name="conditional_process">
+			<param name="select_process" type="select" label="Select process">
+				<option value="calculate-band">calculate-band: Performs a calculation with bands.</option>
+				<option value="download-band-sentinel2-product-safe">download-band-sentinel2-product-safe: Downloads Copernicus products.</option>
+				<option value="plot-image">plot-image: Plots an image with colors.</option>
+				<option value="reproject-image">reproject-image: Performs image reprojection.</option>
+				<option value="select-products-sentinel2">select-products-sentinel2: Searches the specified catalog for Sentinel-2 products matching filtering criteria.</option>
+			</param>
+			<when value="calculate-band">
+				<param name="cookie" label="cookie" optional="false" value="" type="text"/>
+				<param name="calc" label="Calculation" optional="false" help="Calculation to perform with band references." type="text"/>
+				<param name="band_a" label="Band A" optional="true" type="data" format="txt"/>
+				<param name="band_b" label="Band B" optional="true" type="data" format="txt"/>
+				<param name="band_c" label="Band C" optional="true" type="data" format="txt"/>
+				<param name="band_d" label="Band D" optional="true" type="data" format="txt"/>
+				<param name="band_e" label="Band E" optional="true" type="data" format="txt"/>
+				<param name="band_f" label="Band F" optional="true" type="data" format="txt"/>
+				<param name="band_g" label="Band G" optional="true" type="data" format="txt"/>
+				<param name="band_h" label="Band H" optional="true" type="data" format="txt"/>
+				<param name="band_i" label="Band I" optional="true" type="data" format="txt"/>
+				<param name="band_j" label="Band J" optional="true" type="data" format="txt"/>
+				<param name="band_k" label="Band K" optional="true" type="data" format="txt"/>
+				<param name="band_l" label="Band L" optional="true" type="data" format="txt"/>
+				<param name="band_m" label="Band M" optional="true" type="data" format="txt"/>
+				<param name="band_n" label="Band N" optional="true" type="data" format="txt"/>
+				<param name="band_o" label="Band O" optional="true" type="data" format="txt"/>
+				<param name="band_p" label="Band P" optional="true" type="data" format="txt"/>
+				<param name="band_q" label="Band Q" optional="true" type="data" format="txt"/>
+				<param name="band_r" label="Band R" optional="true" type="data" format="txt"/>
+				<param name="band_s" label="Band S" optional="true" type="data" format="txt"/>
+				<param name="band_t" label="Band T" optional="true" type="data" format="txt"/>
+				<param name="band_u" label="Band U" optional="true" type="data" format="txt"/>
+				<param name="band_v" label="Band V" optional="true" type="data" format="txt"/>
+				<param name="band_w" label="Band W" optional="true" type="data" format="txt"/>
+				<param name="band_x" label="Band X" optional="true" type="data" format="txt"/>
+				<param name="band_y" label="Band Y" optional="true" type="data" format="txt"/>
+				<param name="band_z" label="Band Z" optional="true" type="data" format="txt"/>
+			</when>
+			<when value="download-band-sentinel2-product-safe">
+				<param name="product_url" label="Product URL" help="S3 URL to a Sentinel-2 product in SAFE format." optional="false" type="data" format="txt"/>
+				<param name="user_credentials" label="User credentials" optional="false" type="data" format="json"/>
+				<param name="cookie" label="cookie" optional="false" value="" type="text"/>
+				<param name="resolution" label="resolution" optional="true" type="select">
+					<option value="10m">10m</option>
+					<option value="20m">20m</option>
+					<option value="60m">60m</option>
+				</param>
+				<param name="band" label="band" optional="false" type="select">
+					<option value="B01">B01</option>
+					<option value="B02">B02</option>
+					<option value="B03">B03</option>
+					<option value="B04">B04</option>
+					<option value="B05">B05</option>
+					<option value="B06">B06</option>
+					<option value="B07">B07</option>
+					<option value="B08">B08</option>
+					<option value="B8A">B8A</option>
+					<option value="B09">B09</option>
+					<option value="B11">B11</option>
+					<option value="B12">B12</option>
+					<option value="AOT">AOT</option>
+					<option value="SCL">SCL</option>
+					<option value="TCI">TCI</option>
+					<option value="WVP">WVP</option>
+				</param>
+				<param name="debug" label="Debug" optional="true" type="boolean"/>
+			</when>
+			<when value="plot-image">
+				<param name="cookie" label="cookie" optional="false" value="" type="text"/>
+				<param name="input_image" label="Input image" optional="false" help="Image to map with the colors bins." type="data" format="txt"/>
+				<param name="color_scale" label="Color mappings to bin scales to apply to the image." optional="false" help="JSON-like array with each item being an array of 2 elements, the first for the bin scale value and the second for the RGB color. Each RGB value can be either a [0-1] floating point or a [0-255] integer. Example: '[ [0, [0,0,0]], [20, [0,0,255]], [50, [255,100,0]] ]'" type="text"/>
+				<param name="output_name" label="Output name" optional="true" type="text"/>
+				<param name="plot_name" label="Plot name" optional="true" type="text"/>
+				<param name="plot_title" label="Plot title" optional="true" type="text"/>
+			</when>
+			<when value="reproject-image">
+				<param name="cookie" label="cookie" optional="false" value="" type="text"/>
+				<param name="input_image" label="input_image" optional="false" type="data" format="txt"/>
+				<param name="output_dimensions_Array_integer" label="output_dimensions" optional="true" help="Please provide comma-seperated values of type integer here." type="text"/>
+				<param name="output_resolution_Array_integer" label="Output resolution" optional="true" help="Please provide comma-seperated values of type integer here." type="text"/>
+			</when>
+			<when value="select-products-sentinel2">
+				<param name="cookie" label="cookie" optional="false" value="" type="text"/>
+				<param name="date" label="Central date" optional="true" help="Date around which ±delta-days will be applied for search. If omitted, 'toi' input must be provided instead." type="text"/>
+				<param name="delta" label="Delta" optional="true" type="integer"/>
+				<param name="toi_Array_text" label="Time of interest" optional="true" help="Start and end date-time strings. Must be provided if 'date' input is omitted. Please provide comma-seperated values of type text here." type="text"/>
+				<param name="aoi" label="Area of interest" optional="false" help="Polygon defining the area of interest." type="data" format="txt"/>
+				<param name="collection" label="Collection" optional="false" type="text"/>
+				<param name="product_level" label="Product level" optional="true" help="Please provide comma-seperated values of type text here." type="select">
+					<option value="L1C">L1C</option>
+					<option value="L2A">L2A</option>
+				</param>
+				<param name="catalog" label="catalog" optional="false" type="select">
+					<option value="copernicus">copernicus</option>
+					<option value="earth-search">earth-search</option>
+				</param>
+				<param name="cloud_cover" label="Cloud cover" optional="true" type="float"/>
+			</when>
+		</conditional>
+	</inputs>
+	<outputs>
+		<data name="output_data" format="txt" label="$select_process"/>
+	</outputs>
+	<expand macro="tests"/>
+	<help>Use the dropdown menu at the top to select the process and then complete the corresponding from, which is different for every process.</help>
+	<expand macro="citations"/>
+</tool>
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/macros.xml	Fri Sep 06 10:30:22 2024 +0000
@@ -0,0 +1,47 @@
+<macros>
+    <xml name="requirements">
+        <requirements>
+            <requirement type="package" version="4.3.1">r-base</requirement>
+            <requirement type="package" version="1.20.4">r-getopt</requirement>
+            <requirement type="package" version="0.2.3">r-httr2</requirement>
+            <requirement type="package" version="1.8.7">r-jsonlite</requirement>
+        </requirements>
+    </xml>
+    <xml name="out_options">
+        <option value="uint8">uint8</option>
+        <option value="uint16">uint16</option>
+        <option value="int16">int16</option>
+        <option value="int32">int32</option>
+        <option value="float">float</option>
+        <option value="double">double</option>
+    </xml>
+    <xml name="format_options">
+        <option value="image/tiff">image/tiff</option>
+        <option value="image/jpeg">image/jpeg</option>
+        <option value="image/png">image/png</option>
+    </xml>
+    <xml name="citations">
+        <citations>
+            <citation type="bibtex">@Manual{httr2, title = {httr2: Perform HTTP Requests and Process the Responses}, author = {Hadley Wickham}, year = {2023}, note = {R package version 1.0.0, https://github.com/r-lib/httr2}, url = {https://httr2.r-lib.org},}</citation>
+            <citation type="doi">10.48550/arXiv.1403.2805</citation>
+        </citations>
+    </xml>
+    <xml name="help">
+        <help>This tool is a wrapper for OGC API Processes (OTB) coming from the Zoo Project (https://zoo-project.github.io/docs/intro.html) and was created using the OGC-API-Process2Galaxy tool (https://github.com/AquaINFRA/OGC-API-Process2Galaxy). Check the README in the repository for more information.</help>
+    </xml>
+    <xml name="tests">
+        <tests>
+            <test expect_failure="true">
+                <param name="select_process" value="download-band-sentinel2-product-safe"/>
+                <param name="user_credentials" value="crim_test_input.json"/>
+                <param name="cookie" value=""/>
+                <param name="product_url" value="product_url.txt"/>
+                <param name="resolution" value="10m"/>
+                <param name="band" value="B04"/>
+                <assert_stderr>
+                    <has_text text="HTTP 401 Unauthorized"/>
+                </assert_stderr>
+            </test>
+        </tests>    
+    </xml>
+</macros>
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/ogc_api_processes_wrapper.R	Fri Sep 06 10:30:22 2024 +0000
@@ -0,0 +1,320 @@
+library("httr2")
+library("jsonlite")
+library("getopt")
+
+cat("start generic wrapper service \n")
+
+remove_null_values <- function(x) {
+  # Check if the input is a list
+  if (is.list(x)) {
+    # Remove NULL values and apply the function recursively to sublists
+    x <- lapply(x, remove_null_values)
+    x <- x[!sapply(x, is.null)]
+  }
+  return(x)
+}
+
+getParameters <- function() {
+  con <- file("inputs.json", "r")
+  lines <- readLines(con)
+  close(con)
+  
+  json_string <- paste(lines, collapse = "\n")
+  json_data <- fromJSON(json_string)
+  
+  # Remove NULL values from json_data
+  cleaned_json_data <- remove_null_values(json_data)
+  return(cleaned_json_data$conditional_process)
+}
+
+parseResponseBody <- function(body) {
+  hex <- c(body)
+  intValues <- as.integer(hex)
+  rawVector <- as.raw(intValues)
+  readableOutput <- rawToChar(rawVector)
+  jsonObject <- jsonlite::fromJSON(readableOutput)
+  return(jsonObject)
+}
+
+getOutputs <- function(inputs, output, server) {
+  url <-
+    paste(paste(server, "/processes/", sep = ""),
+          inputs$select_process,
+          sep = "")
+  request <- request(url)
+  response <- req_perform(request)
+  responseBody <- parseResponseBody(response$body)
+  outputs <- list()
+  
+  for (x in 1:length(responseBody$outputs)) {
+    outputformatName <-
+      paste(names(responseBody$outputs[x]), "_outformat", sep = "")
+    output_item <- list()
+    
+    for (p in names(inputs)) {
+      if (p == outputformatName) {
+        format <- list("mediaType" = inputs[[outputformatName]])
+        output_item$format <- format
+      }
+    }
+    output_item$transmissionMode <- "reference"
+    outputs[[x]] <- output_item
+  }
+  
+  names(outputs) <- names(responseBody$outputs)
+  return(outputs)
+}
+
+executeProcess <- function(url, process, requestBodyData, cookie) {
+  url <-
+    paste(paste(paste(url, "processes/", sep = ""), process, sep = ""), "/execution", sep = "")
+  requestBodyData$inputs$cookie <- NULL
+  requestBodyData$inputs$select_process <- NULL
+  
+  requestBodyData$inputs$s3_access_key <-
+    requestBodyData$inputs$user_credentials$s3_access_key
+  requestBodyData$inputs$s3_secret_key <-
+    requestBodyData$inputs$user_credentials$s3_secret_key
+  requestBodyData$inputs$user_credentials <- NULL
+  if (process == "plot-image") {
+    tmp <- requestBodyData$inputs$color_scale
+    color_scale <- gsub("__ob__", "[", tmp)
+    color_scale <- gsub("__cb__", "]", color_scale)
+    requestBodyData$inputs$color_scale <- color_scale
+    #print(requestBodyData$inputs$color_scale)
+  }
+  if (process == "calculate-band") {
+    requestBodyData$inputs$name <- "output"
+  }
+  if (process == "reproject-image") {
+    requestBodyData$inputs$output_name <- "output"
+  }
+  #requestBodyData$inputs$input_image$href <- "https://hirondelle.crim.ca/wpsoutputs/weaver/public/test-data/S2A_MSIL2A_20190701T110621_N0500_R137_T29SPC_20230604T023542_turbidity.tiff"
+  
+  body <- list()
+  body$inputs <- requestBodyData$inputs
+  #print(body$inputs)
+  body$mode <- "async"
+  body$response <- "document"
+  #print(body$inputs)
+  
+  response <- request(url) %>%
+    req_headers("Accept" = "application/json",
+                "Content-Type" = "application/json",
+                "Cookie" = cookie) %>%
+    req_body_json(body) %>%
+    req_perform()
+  
+  cat("\n Process executed")
+  cat("\n status: ", response$status_code)
+  cat("\n jobID: ", parseResponseBody(response$body)$jobID, "\n")
+  
+  jobID <- parseResponseBody(response$body)$jobID
+  
+  return(jobID)
+}
+
+checkJobStatus <- function(server, process, jobID, cookie) {
+  url <- paste0(server, "processes/", process, "/jobs/", jobID)
+  response <- request(url) %>%
+    req_headers("Cookie" = cookie) %>%
+    req_perform()
+  jobStatus <- parseResponseBody(response$body)$status
+  jobProgress <- parseResponseBody(response$body)$progress
+  return(jobStatus)
+}
+
+getStatusCode <- function(server, process, jobID, cookie) {
+  url <- paste0(server, "processes/", process, "/jobs/", jobID)
+  response <- request(url) %>%
+    req_headers("Cookie" = cookie) %>%
+    req_perform()
+  status_code <- response$status_code
+  return(status_code)
+}
+
+getResult <- function (server, process, jobID, cookie) {
+  response <-
+    request(paste0(server, "processes/", process, "/jobs/", jobID, "/results")) %>%
+    req_headers("Cookie" = cookie) %>%
+    req_perform()
+  return(response)
+}
+
+retrieveResults <-
+  function(server, process, jobID, outputData, cookie) {
+    status_code <- getStatusCode(server, process, jobID, cookie)
+    if (status_code == 200) {
+      status <- "running"
+      while (status == "running") {
+        jobStatus <- checkJobStatus(server, process, jobID, cookie)
+        print(jobStatus)
+        if (jobStatus == "succeeded") {
+          status <- jobStatus
+          result <- getResult(server, process, jobID, cookie)
+          if (result$status_code == 200) {
+            resultBody <- parseResponseBody(result$body)
+            #print(resultBody)
+            if (process == "select-products-sentinel2") {
+              urls <- unname(unlist(lapply(resultBody, function(x)
+                x$value)))
+            } else if (process == "download-band-sentinel2-product-safe" ||
+                       process == "calculate-band" ||
+                       process == "plot-image" || process == "reproject-image") {
+              urls <- unname(unlist(lapply(resultBody, function(x)
+                x$href)))
+            }
+            urls_with_newline <- paste(urls, collapse = "\n")
+            con <- file(outputData, "w")
+            writeLines(urls_with_newline, con = con)
+            close(con)
+          }
+        } else if (jobStatus == "failed") {
+          status <- jobStatus
+        }
+        Sys.sleep(3)
+      }
+      cat("\n done \n")
+    } else if (status_code1 == 400) {
+      print("A query parameter has an invalid value.")
+    } else if (status_code1 == 404) {
+      print("The requested URI was not found.")
+    } else if (status_code1 == 500) {
+      print("The requested URI was not found.")
+    } else {
+      print(paste("HTTP", status_code1, "Error:", resp1$status_message))
+    }
+  }
+
+is_url <- function(x) {
+  grepl("^https?://", x)
+}
+
+server <- "https://hirondelle.crim.ca/weaver/"
+
+print("--> Retrieve parameters")
+inputParameters <- getParameters()
+#print(inputParameters)
+print("--> Parameters retrieved")
+
+args <- commandArgs(trailingOnly = TRUE)
+outputLocation <- args[2]
+
+print("--> Retrieve outputs")
+outputs <- getOutputs(inputParameters, outputLocation, server)
+print("--> Outputs retrieved")
+
+print("--> Parse inputs")
+convertedKeys <- c()
+for (key in names(inputParameters)) {
+  if (is.character(inputParameters[[key]]) &&
+      (endsWith(inputParameters[[key]], ".dat") ||
+       endsWith(inputParameters[[key]], ".txt"))) {
+    con <- file(inputParameters[[key]], "r")
+    url_list <- list()
+    #while (length(line <- readLines(con, n = 1)) > 0) {
+    #  if (is_url(line)) {
+    #    url_list <- c(url_list, list(list(href = trimws(line))))
+    #  }
+    #}
+    con <- file(inputParameters[[key]], "r")
+    lines <- readLines(con)
+    print("--------------------------------------------------------------------1")
+    print(length(lines))
+    close(con)
+    if (!length(lines) > 1 && endsWith(lines, ".jp2") && startsWith(lines, "https")) {
+      print("--------------------------------------------------------------------2")
+      tmp <- list()
+      tmp$href <- lines
+      tmp$type <- "image/jp2"
+      inputParameters[[key]] <- tmp
+    }
+    else if (!length(lines) > 1 && endsWith(lines, ".SAFE") && startsWith(lines, "s3:")) {
+      print("--------------------------------------------------------------------3")
+      json_string <- paste(lines, collapse = "\n")
+      inputParameters[[key]] <- json_string
+    } else if (inputParameters$select_process == "plot-image" ||
+               inputParameters$select_process == "reproject-image") {
+      print("--------------------------------------------------------------------4")
+      tmp <- list()
+      tmp$href <- lines
+      tmp$type <- "image/tiff; application=geotiff"
+      if (inputParameters$select_process == "reproject-image") {
+        tmp$type <- "image/tiff; subtype=geotiff"
+      }
+      inputParameters[[key]] <- tmp
+    } else {
+      print("-----------------------------------5")
+      json_string <- paste(lines, collapse = "\n")
+      json_data <- fromJSON(json_string)
+      inputParameters[[key]] <- json_data
+    }
+    convertedKeys <- append(convertedKeys, key)
+  }
+  else if (grepl("_Array_", key)) {
+    keyParts <- strsplit(key, split = "_")[[1]]
+    type <- keyParts[length(keyParts)]
+    values <- inputParameters[[key]]
+    value_list <- strsplit(values, split = ",")
+    convertedValues <- c()
+    
+    for (value in value_list) {
+      if (type == "integer") {
+        value <- as.integer(value)
+      } else if (type == "numeric") {
+        value <- as.numeric(balue)
+      } else if (type == "character") {
+        value <- as.character(value)
+      }
+      convertedValues <- append(convertedValues, value)
+      
+      convertedKey <- ""
+      for (part in keyParts) {
+        if (part == "Array") {
+          break
+        }
+        convertedKey <-
+          paste(convertedKey, paste(part, "_", sep = ""), sep = "")
+      }
+      convertedKey <- substr(convertedKey, 1, nchar(convertedKey) - 1)
+    }
+    
+    inputParameters[[key]] <- convertedValues
+    #print("-------------------------")
+    #print(convertedValues)
+    #print("-------------------------")
+    convertedKeys <- append(convertedKeys, convertedKey)
+  } else {
+    #print("-------------------------")
+    #print(key)
+    #print(inputParameters[[key]])
+    if (!is.null(inputParameters[[key]])) {
+      convertedKeys <- append(convertedKeys, key)
+    }
+    #print("-------------------------")
+    
+  }
+}
+#print(inputParameters)
+names(inputParameters) <- convertedKeys
+#print(inputParameters)
+print("--> Inputs parsed")
+
+print("--> Prepare process execution")
+jsonData <- list("inputs" = inputParameters,
+                 "outputs" = outputs)
+
+cookie <- inputParameters$cookie
+
+print("--> Execute process")
+jobID <-
+  executeProcess(server, inputParameters$select_process, jsonData, cookie)
+print("--> Process executed")
+
+print("--> Retrieve results")
+retrieveResults(server,
+                inputParameters$select_process,
+                jobID,
+                outputLocation,
+                cookie)
+print("--> Results retrieved")
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/crim_test_input.json	Fri Sep 06 10:30:22 2024 +0000
@@ -0,0 +1,4 @@
+{
+    "s3_access_key": "1234",
+    "s3_secret_key": "abcd"
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/product_url.txt	Fri Sep 06 10:30:22 2024 +0000
@@ -0,0 +1,1 @@
+s3:///eodata/Sentinel-2/MSI/L2A/2019/07/01/S2A_MSIL2A_20190701T110621_N0212_R137_T29SPC_20190701T120906.SAFE
\ No newline at end of file