I have deployed my application on ecs using fargate instance via template and added lambda function, load balancer to it. We will use, access key, secret access key, region and session token to configure aws. I need to pass those info in my code. Can anyone please guide me on this?
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
import os
import records
import ujson as json
from pycorenlp import StanfordCoreNLP
from corenlp1 import StanfordCoreNLP1
from tqdm import tqdm
import copy
from sqlova_master.wikisql.lib.common import count_lines, detokenize
from sqlova_master.wikisql.lib.query import Query
import boto3
function_name = 'corenlpfunction123'
aws_access_key = 'AKIAVHYFZXFR3GW6DVTK'
aws_secret_key = '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'
region_name = 'us-east-1'
lambda_client = boto3.client('lambda',
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_key,
region_name=region_name)
def annotate(sentence, lower=True):
payload = {
'body': sentence
}
# Invoke Lambda function
response = lambda_client.invoke(
FunctionName=function_name,
InvocationType='RequestResponse', # Use 'Event' for asynchronous invocation
Payload=json.dumps(payload)
)
# Process Lambda response
response_payload = json.loads(response['Payload'].read().decode('utf-8'))
output1 = response_payload
output = json.loads(output1)
words, gloss, after = [], [], []
for s in output['sentences']:
for t in s['tokens']:
words.append(t['word'])
gloss.append(t['originalText'])
after.append(t['after'])
if lower:
words = [w.lower() for w in words]
return {
'gloss': gloss,
'words': words,
'after': after,
}
def annotate_example(example, table):
ann = {'table_id': example['table_id']}
ann['question'] = annotate(example['question'])
ann['table'] = {
'header': [annotate(h) for h in table['header']],
}
ann['query'] = sql = copy.deepcopy(example['sql'])
for c in ann['query']['conds']:
c[-1] = annotate(str(c[-1]))
q1 = 'SYMSELECT SYMAGG {} SYMCOL {}'.format(Query.agg_ops[sql['agg']], table['header'][sql['sel']])
q2 = ['SYMCOL {} SYMOP {} SYMCOND {}'.format(table['header'][col], Query.cond_ops[op], detokenize(cond)) for col, op, cond in sql['conds']]
if q2:
q2 = 'SYMWHERE ' + ' SYMAND '.join(q2) + ' SYMEND'
else:
q2 = 'SYMEND'
inp = 'SYMSYMS {syms} SYMAGGOPS {aggops} SYMCONDOPS {condops} SYMTABLE {table} SYMQUESTION {question} SYMEND'.format(
syms=' '.join(['SYM' + s for s in Query.syms]),
table=' '.join(['SYMCOL ' + s for s in table['header']]),
question=example['question'],
aggops=' '.join([s for s in Query.agg_ops]),
condops=' '.join([s for s in Query.cond_ops]),
)
ann['seq_input'] = annotate(inp)
out = '{q1} {q2}'.format(q1=q1, q2=q2) if q2 else q1
ann['seq_output'] = annotate(out)
ann['where_output'] = annotate(q2)
assert 'symend' in ann['seq_output']['words']
assert 'symend' in ann['where_output']['words']
return ann
def find_sub_list(sl, l):
# from stack overflow.
results = []
sll = len(sl)
for ind in (i for i, e in enumerate(l) if e == sl[0]):
if l[ind:ind + sll] == sl:
results.append((ind, ind + sll - 1))
return results
def check_wv_tok_in_nlu_tok(wv_tok1, nlu_t1):
"""
Jan.2019: Wonseok
Generate SQuAD style start and end index of wv in nlu. Index is for of after WordPiece tokenization.
Assumption: where_str always presents in the nlu.
return:
st_idx of where-value string token in nlu under CoreNLP tokenization scheme.
"""
g_wvi1_corenlp = []
nlu_t1_low = [tok.lower() for tok in nlu_t1]
for i_wn, wv_tok11 in enumerate(wv_tok1):
wv_tok11_low = [tok.lower() for tok in wv_tok11]
results = find_sub_list(wv_tok11_low, nlu_t1_low)
st_idx, ed_idx = results[0]
g_wvi1_corenlp.append( [st_idx, ed_idx] )
return g_wvi1_corenlp
def annotate_example_ws(example):
"""
Jan. 2019: Wonseok
Annotate only the information that will be used in our model.
"""
ann = {'table_id': example['table_id'],'phase': example['phase']}
_nlu_ann = annotate(example['question'])
ann['question'] = example['question']
ann['question_tok'] = _nlu_ann['gloss']
ann['sql'] = example['sql']
ann['query'] = sql = copy.deepcopy(example['sql'])
conds1 = ann['sql']['conds']
wv_ann1 = []
for conds11 in conds1:
_wv_ann1 = annotate(str(conds11[2]))
wv_ann11 = _wv_ann1['gloss']
wv_ann1.append( wv_ann11 )
# Check whether wv_ann exsits inside question_tok
try:
wvi1_corenlp = check_wv_tok_in_nlu_tok(wv_ann1, ann['question_tok'])
ann['wvi_corenlp'] = wvi1_corenlp
except:
ann['wvi_corenlp'] = None
ann['tok_error'] = 'SQuAD style st, ed are not found under CoreNLP.'
return ann
def is_valid_example(e):
if not all([h['words'] for h in e['table']['header']]):
return False
headers = [detokenize(h).lower() for h in e['table']['header']]
if len(headers) != len(set(headers)):
return False
input_vocab = set(e['seq_input']['words'])
for w in e['seq_output']['words']:
if w not in input_vocab:
print('query word "{}" is not in input vocabulary.\n{}'.format(w, e['seq_input']['words']))
return False
input_vocab = set(e['question']['words'])
for col, op, cond in e['query']['conds']:
for w in cond['words']:
if w not in input_vocab:
print('cond word "{}" is not in input vocabulary.\n{}'.format(w, e['question']['words']))
return False
return True
def create_token_file(split,json_file):
'''
This function creates token.jsonl file for prediction
'''
din = './sqlova_master/test_result'
dout = './sqlova_master/test_result'
answer_toy = not True
toy_size = 10
if not os.path.isdir(dout):
os.makedirs(dout)
fsplit = os.path.join(din, split) + '.jsonl'
fout = os.path.join(dout, split) + '_tok.jsonl'
print('Annotating {}'.format(fsplit))
with open(fsplit) as fs, open(fout, 'w+') as fo:
print('Loading tables')
tables = json_file
print('Loading examples')
n_written = 0
cnt = -1
for line in tqdm(fs, total=count_lines(fsplit)):
cnt += 1
d = json.loads(line)
ann = annotate_example_ws(d)
fo.write(json.dumps(ann) + '\n')
n_written += 1
if answer_toy and cnt > toy_size :
break
print('Wrote {} examples'.format(n_written))
Yes, I am aware about that. We wiil remove that once it's working but I need to pass function app url and it should invoke the lambda function. How to pass in my code?
May I ask why you are passing access keys and other information to the Lambda function?