Hello,
I am trying to do a simple POC to use tus for file uploads to S3. The idea is to have the tus server running on EKS behind the AWS API gateway (for the moment, I am doing a POC with EC2 isntead of EKS). The key point here is that AWS API gateway has a timeout of 30 seconds, and I wanted to see if tus would be able to work with that and resume uploads across multiple connections (for big file uploads).
On the tech setup, I deploy a SpringBoot app as my tus server on EC2. I setup API gateway to point to my endpoint on EC2, and enable post, patch, delete, get, head for the API gateway. The API gateway itself works for a dummy GET request that I have setup to make sure I can reach my service (the /ping route in the code). However, when I upload (using a POST via the /upload route in the code), I get the following error IMMEDIATELY (so, not gateway timeout related):
Upload at 0.00 %.
Upload at 100.00 %.
Exception in thread “main” java.lang.reflect.InvocationTargetException
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at org.springframework.boot.loader.MainMethodRunner.run(MainMethodRunner.java:49)
at org.springframework.boot.loader.Launcher.launch(Launcher.java:109)
at org.springframework.boot.loader.Launcher.launch(Launcher.java:58)
at org.springframework.boot.loader.JarLauncher.main(JarLauncher.java:88)
Caused by: io.tus.java.client.ProtocolException: unexpected status code (404) while uploading chunk
at io.tus.java.client.TusUploader.finishConnection(TusUploader.java:281)
at io.tus.java.client.TusUploader.finish(TusUploader.java:267)
at com.blabla.Client$1.makeAttempt(Client.java:39)
at io.tus.java.client.TusExecutor.makeAttempts(TusExecutor.java:85)
at com.blabla.Client.main(Client.java:43)
… 8 more
I have tested uploading a file from the client to server directly without the API gateway, and it works perfectly. Does anyone know what I might be missing? Thank you in advance for your help.
The source code for client and server are attached below:
package com.blabla;
import java.io.IOException;
import java.net.URI;
import java.nio.file.Paths;
import io.tus.java.client.ProtocolException;
import io.tus.java.client.TusClient;
import io.tus.java.client.TusExecutor;
import io.tus.java.client.TusURLMemoryStore;
import io.tus.java.client.TusUpload;
import io.tus.java.client.TusUploader;
import org.springframework.boot.autoconfigure.SpringBootApplication;
SpringBootApplication
public class Client {
public static void main(String args[]) throws IOException, ProtocolException {
int chunkSize = 8*1024;
String endpoint = args[0];
String filepath = args[1];
TusClient client = new TusClient();
client.setUploadCreationURL(URI.create(endpoint).toURL());
client.enableResuming(new TusURLMemoryStore());
TusUpload upload = new TusUpload(Paths.get(filepath).toFile());
TusExecutor executor = new TusExecutor() {
@Override
protected void makeAttempt() throws ProtocolException, IOException {
TusUploader uploader = client.resumeOrCreateUpload(upload);
uploader.setChunkSize(chunkSize);
do {
long totalBytes = upload.getSize();
long bytesUploaded = uploader.getOffset();
double progress = (double) bytesUploaded / totalBytes * 100;
System.out.printf(“Upload at %6.2f %%.\n”, progress);
}
while (uploader.uploadChunk() > -1);
uploader.finish();
}
};
boolean success = executor.makeAttempts();
if (success) {
System.out.println(“Upload successful”);
}
else {
System.out.println(“Upload interrupted”);
}
}
}
package com.blabla;
import java.io.;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.web.bind.annotation.;
import me.desair.tus.server.TusFileUploadService;
import me.desair.tus.server.exception.TusException;
import me.desair.tus.server.upload.UploadInfo;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
RestController
CrossOrigin(exposedHeaders = { “Location”, “Upload-Offset” })
public class UploadController {
private final Regions _regions = Regions.AP_SOUTHEAST_1;
private final TusFileUploadService tusFileUploadService;
// private final Path uploadDirectory;
private final Path tusUploadDirectory;
private final String _bucketName;
private final AmazonS3 _s3client;
public UploadController(TusFileUploadService tusFileUploadService, AppProperties appProps) {
this.tusFileUploadService = tusFileUploadService;
this.tusUploadDirectory = Paths.get(appProps.getTusUploadDirectory());
AWSCredentials credentials = new BasicAWSCredentials(appProps.getAwsAccessKey(), appProps.getAwsSecretKey());
AWSStaticCredentialsProvider provider = new AWSStaticCredentialsProvider(credentials);
_s3client = AmazonS3ClientBuilder.standard().withCredentials(provider).withRegion(_regions).build();
_bucketName = appProps.getAwsBucketName();
}
GetMapping(value = “/ping”)
public String pingConnection() {
return “Connection Ok”;
}
RequestMapping(value = { “/upload”, “/upload/**” }, method = {
RequestMethod.POST, RequestMethod.PATCH,
RequestMethod.HEAD, RequestMethod.DELETE, RequestMethod.GET
})
public void upload(HttpServletRequest servletRequest,
HttpServletResponse servletResponse) throws IOException {
this.tusFileUploadService.process(servletRequest, servletResponse);
String uploadURI = servletRequest.getRequestURI();
UploadInfo uploadInfo = null;
try {
uploadInfo = this.tusFileUploadService.getUploadInfo(uploadURI);
}
catch (IOException | TusException e) {
Application.logger.error(“get upload info”, e);
}
if (uploadInfo != null && !uploadInfo.isUploadInProgress()) {
try (InputStream is = this.tusFileUploadService.getUploadedBytes(uploadURI)) {
// Path output = this.uploadDirectory.resolve(uploadInfo.getFileName());
// Files.copy(is, output, StandardCopyOption.REPLACE_EXISTING);
String ingestedData = _processData(is);
_s3client.putObject(_bucketName, uploadInfo.getFileName(), ingestedData);
}
catch (AmazonServiceException e) {
Application.logger.error(e.getErrorMessage());
}
catch (IOException | TusException e) {
Application.logger.error(“get uploaded bytes”, e);
}
try {
this.tusFileUploadService.deleteUpload(uploadURI);
}
catch (IOException | TusException e) {
Application.logger.error(“delete upload”, e);
}
}
}
private String _processData(InputStream is) throws IOException {
BufferedReader bufferReader = new BufferedReader(new InputStreamReader(is));
StringBuffer dataBuffer = new StringBuffer();
String str;
while((str = bufferReader.readLine())!= null){
dataBuffer.append(str + “\n”);
}
return dataBuffer.toString();
}
Scheduled(fixedDelayString = “PT24H”)
private void cleanup() {
Path locksDir = this.tusUploadDirectory.resolve(“locks”);
if (Files.exists(locksDir)) {
try {
this.tusFileUploadService.cleanup();
}
catch (IOException e) {
Application.logger.error(“error during cleanup”, e);
}
}
}
}