Introduction
This article will demonstrate how to securely upload and delete files from Amazon S3 storage using signed urls using the golang sdk.
Amazon Simple Storage Service (Amazon S3) is an object storage service.
Prerequsites
- An aws s3 bucket where we will be storing our files.
- An access key and secret access key that have been granted to a user with the necessary permission to upload and delete objects from the s3 bucket.
Benefits of Using Presinged URLS
- Controlled Access - set an expiration time for the URL, ensuring temporary access
- Security - accesss is limited to specific s3 objects, reducing the risk of unauthorized access
Demonstration
We will create a REST api that has two routes, upload route that receives a file that will be resized and uploaded to the bucket and delete which will take in the filename to be deleted and delete it.
Setup the API
Create a go project using go mod init <name_of_project>. Once done create a main.go file and create the following functions.
func main() {
fmt.Println("hello world")
setupRoutes()
}
func setupRoutes() {
http.HandleFunc("/upload", UploadHandler())
http.HandleFunc("/delete", DeleteHandler())
http.ListenAndServe(":8080", nil)
}
func UploadHandler() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
}
}
func DeleteHandler() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
}
}
We have setup an http routes to handle the delete and upload action for our project
Setup Environment Variables
We are going to load our environment variables from a .env file. Create .env file in the root of the project and fill it with bucket details in this format
AWS_BUCKET_NAME="<bucket-name-here>"
AWS_REGION="<aws-region-here>"
AWS_S3_BUCKET_ACCESS_KEY="<aws-bucket-access-key-here>"
AWS_S3_BUCKET_SECRET_ACCESS_KEY="<aws-bucket-secret-key-here>"
Once environment variables are setup we need load them into our project. For this this i will use viper
Create a folder named utils. Inside the folder create a file named config.go
Update the file to be
package utils
import (
"log"
"path/filepath"
"github.com/spf13/viper"
)
type Config struct {
AWS_BUCKET_NAME string `mapstructure:"AWS_BUCKET_NAME"`
AWS_REGION string `mapstructure:"AWS_REGION"`
AWS_S3_BUCKET_ACCESS_KEY string `mapstructure:"AWS_S3_BUCKET_ACCESS_KEY"`
AWS_S3_BUCKET_SECRET_ACCESS_KEY string `mapstructure:"AWS_S3_BUCKET_SECRET_ACCESS_KEY"`
}
func LoadViperEnvironment(path string) (config Config, err error) {
viper.SetConfigFile(filepath.Join(path, ".env"))
viper.SetConfigType("env")
// Read the config file
if err := viper.ReadInConfig(); err != nil {
log.Fatalf("Error reading config file, %s", err)
}
err = viper.Unmarshal(&config)
if err != nil {
log.Fatalf("Unable to decode into struct, %v", err)
}
return config, nil
}
We will use the code above to load the environment variables into our routes by updating setupRoutes() in main.go
func setupRoutes() {
// Load configurations from environment variables from .env
// at root of project
cwd, err := os.Getwd()
if err != nil {
log.Fatalf("Error getting current working directory: %v", err)
}
config, err := utils.LoadViperEnvironment(cwd)
if err != nil {
log.Fatalf("Error loading config: %s", err)
}
http.HandleFunc("/upload", UploadHandler(config))
http.HandleFunc("/delete", DeleteHandler(config))
http.ListenAndServe(":8080", nil)
}
We updated setupRoutes to import the app configs from the .env file and then pass those configs to the separate routes where they will be used when creating s3 bucket presigned urls.
Setup S3 bucket Client
Create another folder in the root directory named repo inside repo we will create a file named repo.go.
For this stage of the project I will need to download the necessary modules from the golang sdk
Write the following commands in the terminal to get the sdk modules.
go get github.com/aws/aws-sdk-go-v2
go get github.com/aws/aws-sdk-go-v2/credentials
go get github.com/aws/aws-sdk-go-v2/service/s3
First lets add the following code into the repo.go file
type Repo struct {
s3Client *s3.Client
s3PresignedClient *s3.PresignClient
}
func NewS3Client(accessKey string, secretKey string, s3BucketRegion string) *Repo {
options := s3.Options{
Region: s3BucketRegion,
Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")),
}
client := s3.New(options, func(o *s3.Options) {
o.Region = s3BucketRegion
o.UseAccelerate = false
})
presignClient := s3.NewPresignClient(client)
return &Repo{
s3Client: client,
s3PresignedClient: presignClient,
}
}
func (repo Repo) PutObject(bucketName string, objectKey string, lifetimeSecs int64) (*v4.PresignedHTTPRequest, error) {
request, err := repo.s3PresignedClient.PresignPutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
}, func(opts *s3.PresignOptions) {
opts.Expires = time.Duration(lifetimeSecs * int64(time.Second))
})
if err != nil {
log.Printf("Couldn't get a presigned request to put %v:%v. Here's why: %v\n",
bucketName, objectKey, err)
}
return request, err
}
func (repo Repo) DeleteObject(bucketName string, objectKey string, lifetimeSecs int64) (*v4.PresignedHTTPRequest, error) {
request, err := repo.s3PresignedClient.PresignDeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
}, func(opts *s3.PresignOptions) {
opts.Expires = time.Duration(lifetimeSecs * int64(time.Second))
})
if err != nil {
log.Printf("Couldn't get a presigned request to delete %v:%v. Here's why: %v\n",
bucketName, objectKey, err)
}
return request, err
}
func (repo Repo) UploadFile(file image.Image, url string) error {
var buf bytes.Buffer
err := jpeg.Encode(&buf, file, nil)
if err != nil {
return nil
}
body := io.Reader(&buf)
request, err := http.NewRequest(http.MethodPut, url, body)
if err != nil {
return err
}
request.Header.Set("Content-Type", "image/jpeg")
client := &http.Client{}
resp, err := client.Do(request)
if err != nil {
log.Println("Error sending request:", err)
return err
}
defer resp.Body.Close()
return err
}
func (repo Repo) DeleteFile(url string) error {
request, err := http.NewRequest(http.MethodDelete, url, nil)
if err != nil {
return err
}
response, err := http.DefaultClient.Do(request)
if err != nil {
return err
}
log.Printf("Delete Request Response status code: %v", response.StatusCode)
return err
}
- Create a struct
Repothat holds our s3client and s3PresingedClient. -
NewS3Clienttakes in accesskey, secretkey, s3bucket region and returns the defined repo struct with all containing the s3 clients - The
PutObjectfunction that creates a presigned url for uploading files into the bucket. The function required the bucketname, objectkey(filename), lifetime(how long will the presinged url will be valid) - The
DeleteObjectfunction creates a presigned url for deleting a file from the bucket. The function required bucketname, objectkey(filename) and lifetime to create return the url. -
UploadFilefunction takes in the url generate from thePutobjectfunction and perform a put request populated by a file. -
DeleteFilefunction takes in the url generated fromDeleteObjectfunction and makes a delete request to the url.
Hooking Up everything
We will now update our routes to use the configuration from .env and upload and delete files using the repo functions
Upload Action
func UploadHandler(config utils.Config) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
file, _, err := r.FormFile("image")
if err != nil {
w.WriteHeader(http.StatusBadRequest)
fmt.Fprintf(w, "Error reading file: %v", err)
return
}
defer file.Close()
imageDoc, _, err := image.Decode(file)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintf(w, "Error encoding image: %v", err)
return
}
repo := repo.NewS3Client(
config.AWS_S3_BUCKET_ACCESS_KEY,
config.AWS_S3_BUCKET_SECRET_ACCESS_KEY,
config.AWS_REGION,
)
// Create singed url used for uploading file
presignedurl, err := repo.PutObject(config.AWS_BUCKET_NAME, 'filename', 60)
if err != nil {
log.Fatalf("Error generating presigned url for put object: %s", err)
}
// Uploading document to S3
err = repo.UploadFile(file, presignedurl.URL)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintf(w, "Error uploading image to S3: %v", err)
return
}
w.WriteHeader(http.StatusOK)
fmt.Fprintf(w, "Successfully uploaded image: %s", filename)
}
}
We take the file from the request and decode it before uploading it to the bucket
Delete Action
func DeleteHandler(config utils.Config) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
// Read request body
type RequestBody struct {
Filename string `json:"filename"`
}
var data RequestBody
if r.Body != nil {
json.NewDecoder(r.Body).Decode(&data)
}
if data.Filename == "" {
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintf(w, "Filename must be provided")
return
}
repo := repo.NewS3Client(
config.AWS_S3_BUCKET_ACCESS_KEY,
config.AWS_S3_BUCKET_SECRET_ACCESS_KEY,
config.AWS_REGION,
)
presignedurl, err := repo.DeleteObject(config.AWS_BUCKET_NAME, data.Filename, 60)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
log.Fatalf("Error generating presigned url for put object: %s", err)
return
}
err = repo.DeleteFile(presignedurl.URL)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
log.Fatalf("Error deleting file from s3 bucket: %v", err)
return
}
w.WriteHeader(200)
fmt.Fprintf(w, "Successfully deleted file: %s", data.Filename)
}
}
The delete action works similar to the upload handler difference is we expect a json of the format
{
"filename": "<filename-here>",
}
The file the specified name will be deleted.
Conclusion
The article has goes through the process of deleting and uploading files by using presigned url. Further Reading Material on this
Top comments (0)