Forum Discussion

Krishna's avatar
Krishna
Valued Contributor
8 months ago

AWS S3 SDK

Hello - Does anyone used the AWS SDK (Dot Net) to download the file to the OS batch harvest folder? using SIC if so, could you please provide high level steps?   Any Help would be appreciated.
Extenders
  • kenostrovsky's avatar
    kenostrovsky
    7 months ago

    Krishna, 

    SIC is needed to establish secure access to the client network. Since AWS is hosted in the cloud (outside of the client network) you can just connect to it directly from OneStream using a Business Rule. you will need to request OneStream support to make the Amazon libraries available on the server side. 

     

    it should probably look something like this:

     

    Imports System
    Imports System.Data
    Imports System.Data.Common
    Imports System.IO
    Imports System.Collections.Generic
    Imports System.Globalization
    Imports System.Linq
    Imports Microsoft.VisualBasic
    Imports System.Windows.Forms
    Imports System.Net.Mail
    Imports OneStream.Shared.Common
    Imports OneStream.Shared.Wcf
    Imports OneStream.Shared.Engine
    Imports OneStream.Shared.Database
    Imports OneStream.Stage.Engine
    Imports OneStream.Stage.Database
    Imports OneStream.Finance.Engine
    Imports OneStream.Finance.Database
    Imports Amazon.S3
     
    Namespace OneStream.BusinessRule.Extender.Retrieve_AWS_files
    Public Class MainClass
     
    Public Function Main(ByVal si As SessionInfo, ByVal globals As BRGlobals, ByVal api As Object, ByVal args As ExtenderArgs) As Object
    Try
    Select Case args.FunctionType
     
    Case Is = ExtenderFunctionType.Unknown, ExtenderFunctionType.ExecuteDataMgmtBusinessRuleStep
     
    Retrieve_Files_From_AWS(si)
     
    End Select
     
    Return Nothing
    Catch ex As Exception
    Throw ErrorHandler.LogWrite(si, New XFException(si, ex))
    End Try
    End Function
     
    Function Retrieve_Files_From_AWS(si As SessionInfo) As List(Of String)
     
    Dim AWS_Client = Me.AWS_Client(si)
     
    Dim BucketName As String = "bucket-name"
    Dim Prefix As String = "some/directory"
     
    'Get filenames from AWS bucket
    Dim AWSDirectoryInfo As New IO.S3DirectoryInfo(AWS_Client, BucketName, Prefix)
     
     
    Dim Harvest_Folder_Path As String = BRApi.Utilities.GetFileShareFolder(si, FileShareFolderTypes.BatchHarvest, Nothing)
     
    For Each file In AWSDirectoryInfo.GetFiles()
     
    file.MoveToLocal(Harvest_Folder_Path & "/" & file.Name)
    Next
     
    AWS_Client.Dispose()
     
    Return Nothing
    End Function
     
     
    Function AWS_Client(si As SessionInfo) As AmazonS3Client
    'AWS Credentials
    Dim AccessKey As String = "ASDLKFASJFA;LDSF;LAJF;LSA"
    Dim SecretKey As String = ";lakdsf;laf;lajds;lfajds;lfl"
     
    'Create Amazon S3 Client Object
    Dim awsClient As New AmazonS3Client(AccessKey, SecretKey, Amazon.RegionEndpoint.USEast1)
     
    Return awsClient
    End Function
     
     
     
     
    End Class
    End Namespace
kenostrovsky's avatar
kenostrovsky
New Contributor III
7 months ago

Krishna, 

SIC is needed to establish secure access to the client network. Since AWS is hosted in the cloud (outside of the client network) you can just connect to it directly from OneStream using a Business Rule. you will need to request OneStream support to make the Amazon libraries available on the server side. 

 

it should probably look something like this:

 

Imports System
Imports System.Data
Imports System.Data.Common
Imports System.IO
Imports System.Collections.Generic
Imports System.Globalization
Imports System.Linq
Imports Microsoft.VisualBasic
Imports System.Windows.Forms
Imports System.Net.Mail
Imports OneStream.Shared.Common
Imports OneStream.Shared.Wcf
Imports OneStream.Shared.Engine
Imports OneStream.Shared.Database
Imports OneStream.Stage.Engine
Imports OneStream.Stage.Database
Imports OneStream.Finance.Engine
Imports OneStream.Finance.Database
Imports Amazon.S3
 
Namespace OneStream.BusinessRule.Extender.Retrieve_AWS_files
Public Class MainClass
 
Public Function Main(ByVal si As SessionInfo, ByVal globals As BRGlobals, ByVal api As Object, ByVal args As ExtenderArgs) As Object
Try
Select Case args.FunctionType
 
Case Is = ExtenderFunctionType.Unknown, ExtenderFunctionType.ExecuteDataMgmtBusinessRuleStep
 
Retrieve_Files_From_AWS(si)
 
End Select
 
Return Nothing
Catch ex As Exception
Throw ErrorHandler.LogWrite(si, New XFException(si, ex))
End Try
End Function
 
Function Retrieve_Files_From_AWS(si As SessionInfo) As List(Of String)
 
Dim AWS_Client = Me.AWS_Client(si)
 
Dim BucketName As String = "bucket-name"
Dim Prefix As String = "some/directory"
 
'Get filenames from AWS bucket
Dim AWSDirectoryInfo As New IO.S3DirectoryInfo(AWS_Client, BucketName, Prefix)
 
 
Dim Harvest_Folder_Path As String = BRApi.Utilities.GetFileShareFolder(si, FileShareFolderTypes.BatchHarvest, Nothing)
 
For Each file In AWSDirectoryInfo.GetFiles()
 
file.MoveToLocal(Harvest_Folder_Path & "/" & file.Name)
Next
 
AWS_Client.Dispose()
 
Return Nothing
End Function
 
 
Function AWS_Client(si As SessionInfo) As AmazonS3Client
'AWS Credentials
Dim AccessKey As String = "ASDLKFASJFA;LDSF;LAJF;LSA"
Dim SecretKey As String = ";lakdsf;laf;lajds;lfajds;lfl"
 
'Create Amazon S3 Client Object
Dim awsClient As New AmazonS3Client(AccessKey, SecretKey, Amazon.RegionEndpoint.USEast1)
 
Return awsClient
End Function
 
 
 
 
End Class
End Namespace