How to pass the Lambda function url and session token in my code?

0

I have deployed my application on ecs using fargate instance via template and added lambda function, load balancer to it. We will use, access key, secret access key, region and session token to configure aws. I need to pass those info in my code. Can anyone please guide me on this?

from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
import os
import records
import ujson as json
from pycorenlp import StanfordCoreNLP
from corenlp1 import StanfordCoreNLP1
from tqdm import tqdm
import copy
from sqlova_master.wikisql.lib.common import count_lines, detokenize
from sqlova_master.wikisql.lib.query import Query
import boto3

function_name = 'corenlpfunction123'
aws_access_key = 'AKIAVHYFZXFR3GW6DVTK'
aws_secret_key = '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'
region_name = 'us-east-1'

lambda_client = boto3.client('lambda',
                             aws_access_key_id=aws_access_key,
                             aws_secret_access_key=aws_secret_key,
                             region_name=region_name)

def annotate(sentence, lower=True):
    
        payload = {
            'body': sentence
        }

        # Invoke Lambda function
        response = lambda_client.invoke(
            FunctionName=function_name,
            InvocationType='RequestResponse',  # Use 'Event' for asynchronous invocation
            Payload=json.dumps(payload)
        )

        # Process Lambda response
        response_payload = json.loads(response['Payload'].read().decode('utf-8'))
        output1 = response_payload
        output = json.loads(output1)

        words, gloss, after = [], [], []

        for s in output['sentences']:
            for t in s['tokens']:
                words.append(t['word'])
                gloss.append(t['originalText'])
                after.append(t['after'])
        if lower:
            words = [w.lower() for w in words]
        return {
            'gloss': gloss,
            'words': words,
            'after': after,
        }

def annotate_example(example, table):
    ann = {'table_id': example['table_id']}
    ann['question'] = annotate(example['question'])
    ann['table'] = {
        'header': [annotate(h) for h in table['header']],
    }
    ann['query'] = sql = copy.deepcopy(example['sql'])
    for c in ann['query']['conds']:
        c[-1] = annotate(str(c[-1]))

    q1 = 'SYMSELECT SYMAGG {} SYMCOL {}'.format(Query.agg_ops[sql['agg']], table['header'][sql['sel']])
    q2 = ['SYMCOL {} SYMOP {} SYMCOND {}'.format(table['header'][col], Query.cond_ops[op], detokenize(cond)) for col, op, cond in sql['conds']]
    if q2:
        q2 = 'SYMWHERE ' + ' SYMAND '.join(q2) + ' SYMEND'
    else:
        q2 = 'SYMEND'
    inp = 'SYMSYMS {syms} SYMAGGOPS {aggops} SYMCONDOPS {condops} SYMTABLE {table} SYMQUESTION {question} SYMEND'.format(
        syms=' '.join(['SYM' + s for s in Query.syms]),
        table=' '.join(['SYMCOL ' + s for s in table['header']]),
        question=example['question'],
        aggops=' '.join([s for s in Query.agg_ops]),
        condops=' '.join([s for s in Query.cond_ops]),
    )
    ann['seq_input'] = annotate(inp)
    out = '{q1} {q2}'.format(q1=q1, q2=q2) if q2 else q1
    ann['seq_output'] = annotate(out)
    ann['where_output'] = annotate(q2)
    assert 'symend' in ann['seq_output']['words']
    assert 'symend' in ann['where_output']['words']
    return ann

def find_sub_list(sl, l):
    # from stack overflow.
    results = []
    sll = len(sl)
    for ind in (i for i, e in enumerate(l) if e == sl[0]):
        if l[ind:ind + sll] == sl:
            results.append((ind, ind + sll - 1))

    return results

def check_wv_tok_in_nlu_tok(wv_tok1, nlu_t1):
    """
    Jan.2019: Wonseok
    Generate SQuAD style start and end index of wv in nlu. Index is for of after WordPiece tokenization.

    Assumption: where_str always presents in the nlu.

    return:
    st_idx of where-value string token in nlu under CoreNLP tokenization scheme.
    """
    g_wvi1_corenlp = []
    nlu_t1_low = [tok.lower() for tok in nlu_t1]
    for i_wn, wv_tok11 in enumerate(wv_tok1):
        wv_tok11_low = [tok.lower() for tok in wv_tok11]
        results = find_sub_list(wv_tok11_low, nlu_t1_low)
        st_idx, ed_idx = results[0]

        g_wvi1_corenlp.append( [st_idx, ed_idx] )

    return g_wvi1_corenlp


def annotate_example_ws(example):
    """
    Jan. 2019: Wonseok
    Annotate only the information that will be used in our model.
    """
    ann = {'table_id': example['table_id'],'phase': example['phase']}
    _nlu_ann = annotate(example['question'])
    ann['question'] = example['question']
    ann['question_tok'] = _nlu_ann['gloss']
    ann['sql'] = example['sql']
    ann['query'] = sql = copy.deepcopy(example['sql'])

    conds1 = ann['sql']['conds']
    wv_ann1 = []
    for conds11 in conds1:
        _wv_ann1 = annotate(str(conds11[2]))
        wv_ann11 = _wv_ann1['gloss']
        wv_ann1.append( wv_ann11 )

        # Check whether wv_ann exsits inside question_tok

    try:
        wvi1_corenlp = check_wv_tok_in_nlu_tok(wv_ann1, ann['question_tok'])
        ann['wvi_corenlp'] = wvi1_corenlp
    except:
        ann['wvi_corenlp'] = None
        ann['tok_error'] = 'SQuAD style st, ed are not found under CoreNLP.'

    return ann


def is_valid_example(e):
    if not all([h['words'] for h in e['table']['header']]):
        return False
    headers = [detokenize(h).lower() for h in e['table']['header']]
    if len(headers) != len(set(headers)):
        return False
    input_vocab = set(e['seq_input']['words'])
    for w in e['seq_output']['words']:
        if w not in input_vocab:
            print('query word "{}" is not in input vocabulary.\n{}'.format(w, e['seq_input']['words']))
            return False
    input_vocab = set(e['question']['words'])
    for col, op, cond in e['query']['conds']:
        for w in cond['words']:
            if w not in input_vocab:
                print('cond word "{}" is not in input vocabulary.\n{}'.format(w, e['question']['words']))
                return False
    return True


def create_token_file(split,json_file):
    '''
    This function creates token.jsonl file for prediction
    '''
    din = './sqlova_master/test_result'
    dout = './sqlova_master/test_result'

    answer_toy = not True
    toy_size = 10

    if not os.path.isdir(dout):
        os.makedirs(dout)
    
    fsplit = os.path.join(din, split) + '.jsonl'
    fout = os.path.join(dout, split) + '_tok.jsonl'

    print('Annotating {}'.format(fsplit))
    with open(fsplit) as fs, open(fout, 'w+') as fo:
        print('Loading tables')
        tables = json_file
        print('Loading examples')
        n_written = 0
        cnt = -1
        for line in tqdm(fs, total=count_lines(fsplit)):
            cnt += 1
            d = json.loads(line)
            ann = annotate_example_ws(d)
            fo.write(json.dumps(ann) + '\n')
            n_written += 1

            if answer_toy and cnt > toy_size :
              break

        print('Wrote {} examples'.format(n_written))

asked 9 months ago547 views
2 Answers
2

If you use AWS IAM roles, you will not need to pass credentials.

However, if there is some reason that you cannot use roles, you can use AWS Secrets Manager to securely pass credentials and configuration to your ECS/Fargate application:

  1. Create a secret in Secrets Manager to store the access key, secret key, region, etc. You can store multiple values in a single secret via key/value pairs.
  2. Grant the ECS task execution role permission to call secretsmanager:GetSecretValue on the secret ARN.
  3. In your code, retrieve the secret:
import boto3

secrets = boto3.client('secretsmanager')
secret_string = secrets.get_secret_value(SecretId='my_secret_arn')['SecretString']
secret_dict = json.loads(secret_string)
access_key = secret_dict['AWS_ACCESS_KEY_ID']
secret_key = secret_dict['AWS_SECRET_ACCESS_KEY']

This keeps the credentials secure and separate from your application code.

For parameters such as region which are less sensitive, these can be stored in AWS Systems Manager Parameter Store which is a very similar pattern.

  1. Use AWS Systems Manager Parameter Store to store the credentials as SecureString parameters - e.g. /myapp/AWS_ACCESS_KEY_ID, /myapp/AWS_SECRET_ACCESS_KEY, etc.
  2. Give the ECS task execution role permissions to call ssm:GetParameters on those specific paths.
  3. In your application code, retrieve the credentials from Parameter Store:
ssm = boto3.client('ssm')
region_name = ssm.get_parameter(Name='/myapp/region_name)['Parameter']['Value']

Let me know if you have any other questions!

profile pictureAWS
answered 9 months ago
0

Hello.
Be sure to mask the access key and secret access key listed in the code.

If it is ECS or Lambda that is running the code, it is not necessary to set the access key.
AWS services can be manipulated by setting the necessary IAM policies for IAM roles.
https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_execution_IAM_role.html
https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html

profile picture
EXPERT
answered 9 months ago
profile pictureAWS
EXPERT
reviewed 9 months ago
profile pictureAWS
EXPERT
reviewed 9 months ago
  • Yes, I am aware about that. We wiil remove that once it's working but I need to pass function app url and it should invoke the lambda function. How to pass in my code?

  • May I ask why you are passing access keys and other information to the Lambda function?

You are not logged in. Log in to post an answer.

A good answer clearly answers the question and provides constructive feedback and encourages professional growth in the question asker.

Guidelines for Answering Questions