Commit 6101578f authored by Vít Novotný's avatar Vít Novotný
Browse files

Add arxmliv_prefix_08_2019_warning_1.json.gz, Posts.V1.2_prefix.json.gz

parent ba38c786

output_data/ARQMath_CLEF2020/Collection/.gitignore

+2 −0
Original line number Diff line number Diff line
@@ -4,3 +4,5 @@ 
/Posts_V1_0_opt.json.gz

/Posts_V1_0_prefix.json.gz

/Posts_V1_0_slt.json.gz

/Posts.V1.2_prefix.json.gz

/votes-qrels.V1.2.tsv

output_data/ARQMath_CLEF2020/Collection/Makefile

+26 −24
Original line number Diff line number Diff line 
.PHONY: all sanity-check



SHELL=/bin/bash



RUN_SCRIPT=cd ../../.. && python3 -m



NUM_LINES=2477489

NUM_LINES=$$((2466080 + 2))



all: votes-qrels.V1.0.tsv Posts_V1_0_latex.json.gz Posts_V1_0_opt.json.gz Posts_V1_0_slt.json.gz Posts_V1_0_prefix.json.gz Posts_V1_0_infix.json.gz

all: votes-qrels.V1.2.tsv Posts.V1.2_latex.json.gz Posts.V1.2_opt.json.gz Posts.V1.2_slt.json.gz Posts.V1.2_prefix.json.gz Posts.V1.2_infix.json.gz



sanity-check:

	test $$(gzip -d < Posts_V1_0_latex.json.gz | awk '{ print $$1 }' | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts_V1_0_latex.json.gz | awk '{ print $$1 }' | sort -u | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts_V1_0_opt.json.gz | awk '{ print $$1 }' | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts_V1_0_opt.json.gz | awk '{ print $$1 }' | sort -u | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts_V1_0_slt.json.gz | awk '{ print $$1 }' | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts_V1_0_slt.json.gz | awk '{ print $$1 }' | sort -u | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts_V1_0_prefix.json.gz | awk '{ print $$1 }' | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts_V1_0_prefix.json.gz | awk '{ print $$1 }' | sort -u | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts_V1_0_infix.json.gz | awk '{ print $$1 }' | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts_V1_0_infix.json.gz | awk '{ print $$1 }' | sort -u | wc -l) = $(NUM_LINES)

	gzip -d < Posts_V1_0_latex.json.gz | grep -q '^"3058136":.*"text:implies", "text:that", "math:f(-y) = -f(y)"'

	gzip -d < Posts_V1_0_opt.json.gz | grep -q '^"3058136":.*"text:implies", "text:that", "math:U!eq\\tU!times\\t0\\t-", "math:U!eq\\tV!y\\t00\\t-", "math:U!eq\\tV!f\\t00\\t-", "math:U!times\\tV!f\\t0\\t0", "math:V!f\\t0!\\t0\\t00", "math:U!times\\tV!y\\t0\\t0", "math:V!y\\t0!\\t0\\t00", "math:U!eq\\tU!times\\t0\\t-", "math:U!eq\\tV!y\\t00\\t-", "math:U!eq\\tV!f\\t00\\t-", "math:U!times\\tV!f\\t0\\t0", "math:V!f\\t0!\\t0\\t00", "math:U!times\\tV!y\\t0\\t0", "math:V!y\\t0!\\t0\\t00"'

	gzip -d < Posts_V1_0_slt.json.gz | grep -q '^"3058136":.*"text:implies", "text:that", "math:V!f\\tM!()1x1\\tn\\t-", "math:V!f\\tV!y\\tnw\\t-", "math:V!f\\t=\\tnn\\t-", "math:M!()1x1\\t=\\tn\\tn", "math:M!()1x1\\tV!f\\tnn\\tn", "math:=\\tV!f\\tn\\tnn", "math:=\\tM!()1x1\\tnn\\tnn", "math:V!f\\tM!()1x1\\tn\\tnnn", "math:V!f\\tV!y\\tnw\\tnnn", "math:M!()1x1\\tV!y\\tw\\tnnnn", "math:V!y\\t0!\\tn\\tnnnnw", "math:M!()1x1\\t0!\\tn\\tnnnn", "math:M!()1x1\\tV!y\\tw\\tn", "math:V!y\\t0!\\tn\\tnw"'

	gzip -d < Posts_V1_0_prefix.json.gz | grep -q '^"3058136":.*"text:implies", "text:that", "math:U!eq", "math:U!times", "math:V!f", "math:V!y", "math:U!times", "math:V!f", "math:V!y"'

	gzip -d < Posts_V1_0_infix.json.gz | grep -q '^"3058136":.*"text:implies", "text:that", "math:(", "math:(", "math:V!f", "math:U!times", "math:V!y", "math:)", "math:U!eq", "math:(", "math:V!f", "math:U!times", "math:V!y", "math:)", "math:)"'



votes-qrels.V1.0.tsv:

	test $$(gzip -d < Posts.V1.2_latex.json.gz | awk '{ print $$1 }' | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts.V1.2_latex.json.gz | awk '{ print $$1 }' | sort -u | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts.V1.2_opt.json.gz | awk '{ print $$1 }' | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts.V1.2_opt.json.gz | awk '{ print $$1 }' | sort -u | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts.V1.2_slt.json.gz | awk '{ print $$1 }' | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts.V1.2_slt.json.gz | awk '{ print $$1 }' | sort -u | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts.V1.2_prefix.json.gz | awk '{ print $$1 }' | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts.V1.2_prefix.json.gz | awk '{ print $$1 }' | sort -u | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts.V1.2_infix.json.gz | awk '{ print $$1 }' | wc -l) = $(NUM_LINES)

	test $$(gzip -d < Posts.V1.2_infix.json.gz | awk '{ print $$1 }' | sort -u | wc -l) = $(NUM_LINES)

	gzip -d < Posts.V1.2_latex.json.gz | grep -q '^"3058136":.*"text:implies", "text:that", "math:f(-y) = -f(y)"'

	gzip -d < Posts.V1.2_opt.json.gz | grep -q '^"3058136":.*"text:implies", "text:that", "math:U!eq\\tU!times\\t0\\t-", "math:U!eq\\tV!y\\t00\\t-", "math:U!eq\\tV!f\\t00\\t-", "math:U!times\\tV!f\\t0\\t0", "math:V!f\\t0!\\t0\\t00", "math:U!times\\tV!y\\t0\\t0", "math:V!y\\t0!\\t0\\t00", "math:U!eq\\tU!times\\t0\\t-", "math:U!eq\\tV!y\\t00\\t-", "math:U!eq\\tV!f\\t00\\t-", "math:U!times\\tV!f\\t0\\t0", "math:V!f\\t0!\\t0\\t00", "math:U!times\\tV!y\\t0\\t0", "math:V!y\\t0!\\t0\\t00"'

	gzip -d < Posts.V1.2_slt.json.gz | grep -q '^"3058136":.*"text:implies", "text:that", "math:V!f\\tM!()1x1\\tn\\t-", "math:V!f\\tV!y\\tnw\\t-", "math:V!f\\t=\\tnn\\t-", "math:M!()1x1\\t=\\tn\\tn", "math:M!()1x1\\tV!f\\tnn\\tn", "math:=\\tV!f\\tn\\tnn", "math:=\\tM!()1x1\\tnn\\tnn", "math:V!f\\tM!()1x1\\tn\\tnnn", "math:V!f\\tV!y\\tnw\\tnnn", "math:M!()1x1\\tV!y\\tw\\tnnnn", "math:V!y\\t0!\\tn\\tnnnnw", "math:M!()1x1\\t0!\\tn\\tnnnn", "math:M!()1x1\\tV!y\\tw\\tn", "math:V!y\\t0!\\tn\\tnw"'

	gzip -d < Posts.V1.2_prefix.json.gz | grep -q '^"3058136":.*"text:implies", "text:that", "math:U!eq", "math:U!times", "math:V!f", "math:V!y", "math:U!times", "math:V!f", "math:V!y"'

	gzip -d < Posts.V1.2_infix.json.gz | grep -q '^"3058136":.*"text:implies", "text:that", "math:(", "math:(", "math:V!f", "math:U!times", "math:V!y", "math:)", "math:U!eq", "math:(", "math:V!f", "math:U!times", "math:V!y", "math:)", "math:)"'



votes-qrels.V1.2.tsv:

	$(RUN_SCRIPT) scripts.xml_to_qrels_tsv



Posts_V1_0_latex.json.gz:

Posts.V1.2_latex.json.gz:

	$(RUN_SCRIPT) scripts.xml_and_csv_to_latex_json



Posts_V1_0_opt.json.gz:

Posts.V1.2_opt.json.gz:

	$(RUN_SCRIPT) scripts.xml_and_csv_to_opt_json



Posts_V1_0_slt.json.gz:

Posts.V1.2_slt.json.gz:

	$(RUN_SCRIPT) scripts.xml_and_csv_to_slt_json



Posts_V1_0_prefix.json.gz:

Posts.V1.2_prefix.json.gz:

	$(RUN_SCRIPT) scripts.xml_and_csv_to_prefix_json



Posts_V1_0_infix.json.gz:

Posts.V1.2_infix.json.gz:

	$(RUN_SCRIPT) scripts.xml_and_csv_to_infix_json

output_data/ARQMath_CLEF2020/Collection/Posts_V1_0_prefix.json.gz.dvc→output_data/ARQMath_CLEF2020/Collection/Posts.V1.2_prefix.json.gz.dvc

+11 −0
Original line number Diff line number Diff line
@@ -3,9 +3,9 @@ deps: 
  path: ../Formulas/formula_prefix.V1.0.tsv

outs:

- cache: true

  path: Posts_V1_0_prefix.json.gz

  path: Posts.V1.2_prefix.json.gz

  metric: false

  persist: false

  md5: 39492079401bdf8e3a579f3417d306bb

md5: 85eb8a8892c215cdc4d290a44c47fe64

cmd: make Posts_V1_0_prefix.json.gz
 
  md5: f250d5702665d6e0853bec79ee7dcc7b

md5: 97b972e2537533f1f72171f09300d9d6

cmd: make Posts.V1.2_prefix.json.gz

output_data/ARQMath_CLEF2020/Collection/Posts_V1_0_infix.json.gz.dvc

deleted100644 → 0
+0 −11
Original line number Diff line number Diff line 
deps:

- md5: 41dd59f2f8a650df2336c0943ab4b019

  path: ../Formulas/formula_infix.V1.0.tsv

md5: 24d3372521479c09923e98004ce39144

outs:

- path: Posts_V1_0_infix.json.gz

  md5: cc790e32bb913c27e28e36afa7e89ec5

  metric: false

  persist: false

  cache: true

cmd: make Posts_V1_0_infix.json.gz

output_data/ARQMath_CLEF2020/Collection/Posts_V1_0_latex.json.gz.dvc

deleted100644 → 0
+0 −8
Original line number Diff line number Diff line 
outs:

- persist: false

  md5: 9e4726636426162683fd5cb2b0a0dcd2

  cache: true

  path: Posts_V1_0_latex.json.gz

  metric: false

md5: 46789bf4fdbf10233d9a4735f940e56a

cmd: make Posts_V1_0_latex.json.gz