An overview of the
As cloud technology becomes more and more mature, more and more companies use cloud technology to store files, and Amazon provides an increasingly mature cloud environment of server farms for convenient storage of files.
Amazon Simple Storage Service (Amazon S3) is an object Storage Service that provides industry-leading scalability, data availability, security, and performance. This means that customers of all sizes and industries can use S3 to store and protect data in unlimited volumes for a variety of use cases, such as data lakes, websites, mobile applications, backup and restore, archiving, enterprise applications, IoT devices, and big data analytics. Amazon S3 provides easy-to-use management capabilities, so you can organize data and configure fine-tuned access controls to meet specific business, organizational, and compliance requirements. Amazon S3 is 99.99999999999% (11 nines) persistent and stores data for millions of applications for companies around the world.
Here is how to upload files to AWS in a NET environment.
Main implementation methods
1. Define interface IRemoteFile.
public interface IRemoteFile
{
string Upload(string sourceFile, string relativePath, string bucketName = null);
bool Delete(string url);
Stream Download(string relativePath, string bucketName);
long ComputeSize(string relativePath, string[] excludeKeywords);
}
Copy the code2. Implementation methods, such as Upload method.
private static NLog.Logger logger = NLog.LogManager.GetCurrentClassLogger(); private static IAmazonS3 client; private string _bucketName = MvcTools.GetAppSetting("AmazonS3Bucket"); public string Upload(string sourceFile, string relativePath, string bucketName = null) { if (bucketName ! = null) { _bucketName = bucketName; } else { _bucketName = MvcTools.GetAppSetting("AmazonS3Bucket"); } string resultUrl = string.Empty; bool success = false; using (client = new AmazonS3Client()) { try { PutObjectResponse putObjectResponse = client.PutObject(new PutObjectRequest { BucketName = _bucketName, FilePath = sourceFile, Grants = new List<S3Grant>() { new S3Grant() { Permission = S3Permission.READ, Grantee = new S3Grantee { URI = "http://acs.amazonaws.com/groups/global/AllUsers" } } }, Key = relativePath, }); success = putObjectResponse.HttpStatusCode == System.Net.HttpStatusCode.OK; if (! success) { logger.Error(putObjectResponse.ToJsonString()); } else { resultUrl = string.Format("http://{0}.s3.amazonaws.com/{1}" , _bucketName , relativePath ); } } catch (AmazonS3Exception ex) { logger.Error(ex.Message); } } return ConvertS3Url(resultUrl); }Copy the code3. Configure AWS information.
<! --AWS start--> <add key="AWSAccessKey" value="xxxx"/> <add key="AWSSecretKey" value="xxxxx"/> <add key="AmazonS3WebBucket" value="xxxx"/> <! --AWS end-->Copy the code4. Call the upload method to upload files.
public string UploadRemote(string filePath, Inner_Download_Report model, IRemoteFile remoteFile)
{
var txtRelativePath = GetRelativeFilePath(model.Locale, model.Account, "Report", SetTxtFileName());
return remoteFile.Upload(filePath, txtRelativePath, MvcTools.GetAppSetting("AmazonS3WebBucket"));
}
Copy the codeThe complete code
public class AmazonS3 : IRemoteFile
{
private static NLog.Logger logger = NLog.LogManager.GetCurrentClassLogger();
private static IAmazonS3 client;
private string _bucketName = MvcTools.GetAppSetting("AmazonS3Bucket");
public string Upload(string sourceFile, string relativePath, string bucketName = null)
{
if (bucketName != null)
{
_bucketName = bucketName;
}
else
{
_bucketName = MvcTools.GetAppSetting("AmazonS3Bucket");
}
string resultUrl = string.Empty;
bool success = false;
using (client = new AmazonS3Client())
{
try
{
PutObjectResponse putObjectResponse = client.PutObject(new PutObjectRequest
{
BucketName = _bucketName,
FilePath = sourceFile,
Grants = new List<S3Grant>() {
new S3Grant()
{
Permission = S3Permission.READ,
Grantee = new S3Grantee { URI = "http://acs.amazonaws.com/groups/global/AllUsers" }
}
},
Key = relativePath,
});
success = putObjectResponse.HttpStatusCode == System.Net.HttpStatusCode.OK;
if (!success)
{
logger.Error(putObjectResponse.ToJsonString());
}
else
{
resultUrl = string.Format("http://{0}.s3.amazonaws.com/{1}"
, _bucketName
, relativePath
);
}
}
catch (AmazonS3Exception ex)
{
logger.Error(ex.Message);
}
}
return ConvertS3Url(resultUrl);
}
public bool Delete(string url)
{
bool success = false;
if (url == null)
{
return false;
}
var (isMatchSuccess, bucketName, filePath) = MatchS3Url(url);
if (!isMatchSuccess)
{
logger.Info("regex match fail, remoteUrl: {0}", url);
}
else
{
using (client = new AmazonS3Client())
{
try
{
DeleteObjectResponse deleteObjectResponse = client.DeleteObject(new DeleteObjectRequest
{
BucketName = bucketName,
Key = filePath,
});
success = deleteObjectResponse.HttpStatusCode == System.Net.HttpStatusCode.NoContent;
if (!success)
{
logger.Info(deleteObjectResponse.ToJsonString());
}
}
catch (AmazonS3Exception ex)
{
logger.Error(ex.Message);
}
}
}
return success;
}
public long ComputeSize(string relativePath, string[] excludeKeywords)
{
using (client = new AmazonS3Client())
{
try
{
ListObjectsResponse listObjectsResponse = client.ListObjects(new ListObjectsRequest
{
BucketName = _bucketName,
Prefix = relativePath,
MaxKeys = int.MaxValue
});
if (listObjectsResponse.HttpStatusCode == System.Net.HttpStatusCode.OK)
{
return listObjectsResponse.S3Objects
.Where(r => NotContainKeyword(excludeKeywords, r.Key))
.Sum(r => r.Size);
}
else
{
logger.Info(listObjectsResponse.ToJsonString());
}
}
catch (AmazonS3Exception ex)
{
logger.Error(ex.Message);
}
}
return 0;
}
private bool NotContainKeyword(string[] keywords, string value)
{
if (keywords == null || keywords.Length == 0)
{
return true;
}
else
{
foreach (var key in keywords)
{
if (value.IndexOf(key) >= 0)
{
return false;
}
}
}
return true;
}
public Stream Download(string relativePath, string bucketName)
{
_bucketName = bucketName;
bool success = false;
GetObjectResponse getObjectResponse = new GetObjectResponse();
using (client = new AmazonS3Client())
{
try
{
GetObjectRequest request = new GetObjectRequest
{
BucketName = bucketName,
Key = relativePath
};
getObjectResponse = client.GetObject(request);
// using (responseStream = getObjectResponse.ResponseStream)
success = getObjectResponse.HttpStatusCode == System.Net.HttpStatusCode.OK;
if (!success)
{
logger.Error(getObjectResponse.ToJsonString());
return null;
}
}
catch (AmazonS3Exception ex)
{
logger.Error(ex.Message);
return null;
}
}
return getObjectResponse.ResponseStream;
}
/// <summary>
/// from: http://(bucket).s3.amazonaws.com/(filePath)
/// to: https://s3-us-west-2.amazonaws.com/(bucket)/(filePath)
/// </summary>
/// <param name="url"></param>
/// <param name="awsRegion"></param>
/// <returns></returns>
public static string ConvertS3Url(string url, string awsRegion = "s3-us-west-2")
{
// from: http://amz.file.s3.amazonaws.com/us/67420000/logo/logo.jpg
// to: https://s3-us-west-2.amazonaws.com/amz.file/us/67420000/logo/logo.jpg
if (string.IsNullOrWhiteSpace(url))
{
return url;
}
var (isMatchSuccess, bucketName, filePath) = MatchS3Url(url, awsRegion);
if (isMatchSuccess)
{
return $"https://{awsRegion}.amazonaws.com/{bucketName}/{filePath}";
}
return url;// fail
}
/// <summary>
/// Get [bucketName, filePath] from url
/// </summary>
/// <param name="url">S3 url format: [http://(bucket).s3.amazonaws.com/(filePath) , https://s3-us-west-2.amazonaws.com/(bucket)/(filePath)]</param>
/// <param name="awsRegion"></param>
/// <returns></returns>
public static (bool isMatchSuccess, string bucketName, string filePath) MatchS3Url(string url, string awsRegion = "s3-us-west-2")
{
List<Regex> partternList = new List<Regex>
{
new Regex("http://(?<bucket>.*?).s3.amazonaws.com/(?<filePath>.*)", RegexOptions.IgnoreCase),
new Regex($"http[s]?://{awsRegion}.amazonaws.com/(?<bucket>.*?)/(?<filePath>.*)", RegexOptions.IgnoreCase),
};
foreach (var parttern in partternList)
{
var match = parttern.Match(url);
if (match.Success)
{
string bucket = match.Groups["bucket"].Value;
string filePath = match.Groups["filePath"].Value;
return (true, bucket, filePath);
}
}
logger.Warn("regex match fail, url: {0}", url);
return (false, null, null);
}
}
Copy the code