Commit 8b27823c authored by Marek Medved's avatar Marek Medved
Browse files

version assigment

parent d158bb49

Makefile

+20 −3
Original line number Diff line number Diff line 
DB_NAME=sqad_db/devel/sqad_v3_$(shell date +"%d-%m-%Y_%H-%M-%S")

#VERSION=$(file < ./sqad_db/version)

VERSION=1

NEW_VERSION=$$(($(VERSION)+1))

create:

	printf "SQAD to DB\n=======================" >> $(DB_NAME).log

	./sqad2database.py -p /nlp/projekty/sqad/sqad_v3/data -n $(DB_NAME) 2>> $(DB_NAME).log

	echo $(NEW_VERSION) > ./sqad_db/version

	./sqad2database.py -p /nlp/projekty/sqad/sqad_v3/data -n $(DB_NAME) -v $(NEW_VERSION) 2>> $(DB_NAME).log

	printf "Contains answer\n======================" >> $(DB_NAME).log

	./add_contains_answer_sentences.py -d $(DB_NAME) 2>> $(DB_NAME).log

	printf "Similar answers\n======================" >> $(DB_NAME).log
@@ -11,7 +15,20 @@ create: 
	printf "Context previous sentece\n======================" >> $(DB_NAME).log

	./context_previous_senteces.py -d $(DB_NAME) 2>> $(DB_NAME).log

	printf "Context wiki entity\n======================" >> $(DB_NAME).log

	conda activate mypy3; python ./context_ner.py -d $(DB_NAME) -m named_entity_recognition/BERT-NER/ner_model_cz/ 2>> $(DB_NAME).log

	#conda activate mypy3; python ./context_ner.py -d $(DB_NAME) -m named_entity_recognition/BERT-NER/ner_model_cz/ 2>> $(DB_NAME).log



run_ZODB_server:

	cd "$(HOME)/.local/lib/python3.6/site-packages/"; exec "/usr/bin/python3.6" -m "ZEO.runzeo" -a "0.0.0.0:9001" -f "/nlp/projekty/question_answering/AQA_v2/sqad_tools/sqad2database/sqad_db/stable"
 
	exec "/usr/bin/python3.6" -m "ZEO.runzeo" -C /nlp/projekty/question_answering/AQA_v2/sqad_tools/sqad2database/zeo_server.conf

	#cd "$(HOME)/.local/lib/python3.6/site-packages/"; exec "/usr/bin/python3.6" -m "ZEO.runzeo" -a "0.0.0.0:9001" -f "/nlp/projekty/question_answering/AQA_v2/sqad_tools/sqad2database/sqad_db/stable"



demo_query:

	./query_database.py -d sqad_db/stable -r 000180 --simple | head -n 38



demo_query_1:

	./query_database.py -d sqad_db/stable -r 000180 | less



demo_zeo:

	exec "/usr/bin/python3.6" -m "ZEO.runzeo" -C /nlp/projekty/question_answering/AQA_v2/sqad_tools/sqad2database/zeo_server_9003.conf



demo_zeo_query:

	./query_database.py -u "0.0.0.0" -p 9003 -r 000180 --simple | head -n 38

add_contains_answer_sentences.py

+3 −0
Original line number Diff line number Diff line
@@ -66,6 +66,9 @@ def main(): 
            record.similar_answers['sents_containing_ans_ext'] = sent_containing_answer

            db._p_changed = True

            transaction.commit()

        db.update()

        db._p_changed = True

        transaction.commit()

        db.close()

    except KeyboardInterrupt:

        db.close()

add_similar_senteces.py

+4 −0
Original line number Diff line number Diff line
@@ -167,6 +167,10 @@ def main(): 
                record.similar_answers[f'sents_similar_{args.number}'] = persistent.list.PersistentList(sorted_sim_sentences[:args.number])

            db._p_changed = True

            transaction.commit()



        db.update()

        db._p_changed = True

        transaction.commit()

        db.close()

    except KeyboardInterrupt:

        db.close()

context_ner.py

+2 −4
Original line number Diff line number Diff line
@@ -38,11 +38,10 @@ def ner_phrases(text, context_window, model): 
        sent_w = ' '.join([x["word"] for x in sent])



        w_idx = 0

        print(f'sent_w: {sent_w}\n')

        ner_out = model.predict(sent_w)

        ner_out = normalize_out(ner_out, sent)



        # sys.stderr.write(f'sent: {sent}\n')

        # sys.stderr.write(f'ner_out: {ner_out}\n')

        while w_idx < len(ner_out):

            if ner_out[w_idx]['tag'] == 'B':

                if w_idx+1 < len(ner_out):
@@ -52,8 +51,6 @@ def ner_phrases(text, context_window, model): 
                    elif ner_out[w_idx+1]['tag'] == 'I':

                        composite_ner = []

                        while w_idx < len(ner_out) and ner_out[w_idx]['tag'] != 'O':

                            # sys.stderr.write(f'w_idx: {w_idx}\n')

                            # sys.stderr.write(f'len: {len(ner_out)}\n')

                            composite_ner.append(sent[w_idx]['id'])

                            w_idx += 1

                        sent_ners.append(composite_ner)
@@ -178,6 +175,7 @@ def main(): 
        add_ner(db, args.context_window, model, args.verbose)



        db.root['__ctx_types__'].append(f'ctx_ner_{args.context_window}')

        db.update()

        db._p_changed = True

        transaction.commit()

        db.close()

context_np.py

+1 −0
Original line number Diff line number Diff line
@@ -202,6 +202,7 @@ def main(): 
        add_np_phrases(db, args.context_window, args.num_phr_per_sent, w2v, args.verbose)



        db.root['__ctx_types__'].append(f'name_phrs_w{args.context_window}_n{args.num_phr_per_sent}')

        db.update()

        db._p_changed = True

        transaction.commit()

        db.close()