collective/wikiq
20
0
Fork 0
Code Issues Pull Requests 3 Activity

write regex captures to parquet arrays.

Browse Source
This commit is contained in:
Nathan TeBlunthuis
2022-03-29 17:52:26 -07:00
parent 32283aa4da
commit b124f9c7c8
9 changed files with 359 additions and 353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
test/baseline_output/basic_regextest_0.tsv
View File

@@ -1,27 +1,27 @@
anon articleid date_time deleted editor editorid minor namespace revert reverteds revid sha1 text_chars threedigits title
FALSE 56237363 2018-01-07 10:40:58 FALSE "NinjaRobotPirate" 3742946 FALSE 3 FALSE 819091731 135nz8q6lfam6cojla7azb7k5alx3t3 1141 126, 126, 126, 126 "User talk:86.139.142.254"
FALSE 56237364 2018-01-07 10:41:10 FALSE "Kavin kavitha" 32792125 FALSE 3 FALSE 819091755 0pwezjc6yopz0smc8al6ogc4fax5bwo 663 "User talk:Kavin kavitha"
FALSE 56237365 2018-01-07 10:41:26 FALSE "Amicable always" 32621254 FALSE 3 FALSE 819091788 sz3t2ap7z8bpkdvdvi195f3i35949bv 399 "User talk:Dr.vivek163"
FALSE 56237366 2018-01-07 10:41:31 FALSE "ClueBot NG" 13286072 FALSE 3 FALSE 819091796 r6s5j8j3iykenrhuhpnkpsmmd71vubf 1260 "User talk:Twistorl"
FALSE 56237368 2018-01-07 10:41:51 FALSE "Khruner" 8409334 FALSE 0 FALSE 819091825 tf5qz2yaswx61zrlm9ovxzuhl7r2dc4 2249 119, 978, 500, 292, 225, 199, 292 "Kom Firin"
FALSE 56237368 2018-01-27 12:16:02 FALSE "Khruner" 8409334 TRUE 0 FALSE 822610647 e6oa4g0qv64icdaq26uu1zzbyr5hcbh 2230 119, 978, 500, 292, 225, 199, 292 "Kom Firin"
FALSE 56237369 2018-01-07 10:42:05 FALSE "Editingaccount1994" 32794215 FALSE 2 FALSE 819091844 0fyvyh2a8xu41gt8obr34oba0bfixj6 27840 798, 150, 150, 150, 621, 137, 137, 150, 150, 350, 195, 350, 195, 180, 180, 350, 195, 300, 150, 150, 150, 180, 180, 621 "User:Editingaccount1994/sandbox"
FALSE 56237369 2018-01-07 11:09:52 FALSE "AnomieBOT" 7611264 TRUE 2 FALSE 819093984 8gy52aolt5rg3eaketwj5v7eiw0apv2 27787 798, 150, 150, 150, 621, 137, 137, 150, 150, 350, 195, 350, 195, 180, 180, 350, 195, 300, 150, 150, 150, 180, 180, 621 "User:Editingaccount1994/sandbox"
FALSE 56237369 2018-01-12 21:45:50 FALSE "SporkBot" 12406635 TRUE 2 FALSE 820064189 he8ydemaanxlrpftqxkez8jfpge1fsj 27784 798, 150, 150, 150, 621, 137, 137, 150, 150, 350, 195, 350, 195, 180, 180, 350, 195, 300, 150, 150, 150, 180, 180, 621 "User:Editingaccount1994/sandbox"
FALSE 56237369 2018-01-12 23:28:11 FALSE "SporkBot" 12406635 TRUE 2 FALSE 820078679 0to17w9rth3url8n7gvucdtobybdq5h 27783 798, 150, 150, 150, 621, 137, 137, 150, 150, 350, 195, 350, 195, 180, 180, 350, 195, 300, 150, 150, 150, 180, 180, 621 "User:Editingaccount1994/sandbox"
FALSE 56237369 2018-01-12 23:28:39 FALSE "SporkBot" 12406635 TRUE 2 FALSE 820078733 531dizmmloyxffbkdr5vph7owh921eg 27782 798, 150, 150, 150, 621, 137, 137, 150, 150, 350, 195, 350, 195, 180, 180, 350, 195, 300, 150, 150, 150, 180, 180, 621 "User:Editingaccount1994/sandbox"
FALSE 56237369 2018-01-13 13:45:33 FALSE "Frietjes" 13791031 FALSE 2 FALSE 820177382 nik9p2u2fuk4yazjxt8ymbicxv5qid9 27757 798, 150, 150, 150, 621, 100, 621 "User:Editingaccount1994/sandbox"
FALSE 56237369 2018-01-24 01:35:22 FALSE "CommonsDelinker" 2304267 FALSE 2 FALSE 822038928 gwk6pampl8si1v5pv3kwgteg710sfw3 27667 798, 150, 150, 150, 621, 100, 621 "User:Editingaccount1994/sandbox"
FALSE 56237370 2018-01-07 10:42:20 FALSE "PamD" 1368779 FALSE 0 FALSE 819091874 n4ozbsgle13p9yywtfrz982ccj8woc9 25 "Anita del Rey"
FALSE 56237371 2018-01-07 10:42:27 FALSE "ClueBot NG" 13286072 FALSE 3 FALSE 819091883 ksohnvsbeuzwpl5vb8a3v8m18hva0a7 1274 119, 157, 119, 157, 119, 157, 119, 157 "User talk:119.94.96.157"
FALSE 56237372 2018-01-07 10:42:50 FALSE "Underbar dk" 677153 FALSE 14 FALSE 819091914 je7aw21fedbwyqsyofpisdrynsu7olr 113 "Category:Ohmi Railway"
FALSE 56237375 2018-01-07 10:43:32 FALSE "TastyPoutine" 882433 FALSE 3 FALSE 819091968 cpm4tkzcx4hc6irr9ukbi06ogud8dtq 199 "User talk:92.226.219.222"
FALSE 56237375 2018-01-07 11:10:24 FALSE "AnomieBOT" 7611264 TRUE 3 FALSE 819094036 artmfz8b2gxhb3pp8a5p4ksplxqfkpg 1840 "User talk:92.226.219.222"
FALSE 56237375 2018-01-07 14:33:36 FALSE "Only" 702940 FALSE 3 FALSE 819112363 dn9wj0n8d8pdd5lqe56uw5xamupowr1 2949 126, 126, 126, 126 "User talk:92.226.219.222"
FALSE 56237376 2018-01-07 10:44:01 FALSE "Dipayanacharya" 32794237 FALSE 2 FALSE 819092004 ofueugwatmmn7u73isw732neuza57gk 28 "User:Dipayanacharya"
FALSE 56237376 2018-01-07 10:49:08 FALSE "Dipayanacharya" 32794237 FALSE 2 FALSE 819092390 dsz55xv96ec2uv6w9c1z7c52ipfovbw 38 "User:Dipayanacharya"
FALSE 56237378 2018-01-07 10:44:56 FALSE "Vinegarymass911" 21516552 FALSE 0 FALSE 819092066 9ma38hak0ef1ew4fpiutxpnzd8oz1wd 65 "BSCIC"
FALSE 56237379 2018-01-07 10:45:21 FALSE "BrownHairedGirl" 754619 FALSE 14 FALSE 819092102 4dvakoat58bzyf5hmtthxukt29hip6n 285 "Category:Women government ministers of Yemen"
FALSE 56237381 2018-01-07 10:45:54 FALSE "PRehse" 410898 FALSE 1 FALSE 819092135 2sjrxsc7os9k9pg4su2t4rk2j8nn0h7 103 "Talk:List of Morning Glories Characters"
FALSE 56237382 2018-01-07 10:45:56 FALSE "ClueBot NG" 13286072 FALSE 3 FALSE 819092138 3y9t5wpk6ur5jhone75rhm4wjf01fgi 1330 106, 207, 126, 114, 106, 207, 126, 114, 106, 207, 126, 114, 106, 207, 126, 114 "User talk:106.207.126.114"
FALSE 56237382 2018-01-07 10:50:22 FALSE "HindWIKI" 31190506 FALSE 3 FALSE 819092495 8wvn6vh3isyt0dorpe89lztrburgupe 2355 106, 207, 126, 114, 106, 207, 126, 114, 106, 207, 126, 114, 106, 207, 126, 114 "User talk:106.207.126.114"
revid date_time articleid editorid title namespace deleted text_chars revert reverteds sha1 minor editor anon threedigits
819091731 2018-01-07 10:40:58 56237363 3742946 "User talk:86.139.142.254" 3 FALSE 1141 FALSE 135nz8q6lfam6cojla7azb7k5alx3t3 FALSE "NinjaRobotPirate" FALSE "126,126,126,126"
819091755 2018-01-07 10:41:10 56237364 32792125 "User talk:Kavin kavitha" 3 FALSE 663 FALSE 0pwezjc6yopz0smc8al6ogc4fax5bwo FALSE "Kavin kavitha" FALSE
819091788 2018-01-07 10:41:26 56237365 32621254 "User talk:Dr.vivek163" 3 FALSE 399 FALSE sz3t2ap7z8bpkdvdvi195f3i35949bv FALSE "Amicable always" FALSE
819091796 2018-01-07 10:41:31 56237366 13286072 "User talk:Twistorl" 3 FALSE 1260 FALSE r6s5j8j3iykenrhuhpnkpsmmd71vubf FALSE "ClueBot NG" FALSE
819091825 2018-01-07 10:41:51 56237368 8409334 "Kom Firin" 0 FALSE 2249 FALSE tf5qz2yaswx61zrlm9ovxzuhl7r2dc4 FALSE "Khruner" FALSE "119,978,500,292,225,199,292"
822610647 2018-01-27 12:16:02 56237368 8409334 "Kom Firin" 0 FALSE 2230 FALSE e6oa4g0qv64icdaq26uu1zzbyr5hcbh TRUE "Khruner" FALSE "119,978,500,292,225,199,292"
819091844 2018-01-07 10:42:05 56237369 32794215 "User:Editingaccount1994/sandbox" 2 FALSE 27840 FALSE 0fyvyh2a8xu41gt8obr34oba0bfixj6 FALSE "Editingaccount1994" FALSE "798,150,150,150,621,137,137,150,150,350,195,350,195,180,180,350,195,300,150,150,150,180,180,621"
819093984 2018-01-07 11:09:52 56237369 7611264 "User:Editingaccount1994/sandbox" 2 FALSE 27787 FALSE 8gy52aolt5rg3eaketwj5v7eiw0apv2 TRUE "AnomieBOT" FALSE "798,150,150,150,621,137,137,150,150,350,195,350,195,180,180,350,195,300,150,150,150,180,180,621"
820064189 2018-01-12 21:45:50 56237369 12406635 "User:Editingaccount1994/sandbox" 2 FALSE 27784 FALSE he8ydemaanxlrpftqxkez8jfpge1fsj TRUE "SporkBot" FALSE "798,150,150,150,621,137,137,150,150,350,195,350,195,180,180,350,195,300,150,150,150,180,180,621"
820078679 2018-01-12 23:28:11 56237369 12406635 "User:Editingaccount1994/sandbox" 2 FALSE 27783 FALSE 0to17w9rth3url8n7gvucdtobybdq5h TRUE "SporkBot" FALSE "798,150,150,150,621,137,137,150,150,350,195,350,195,180,180,350,195,300,150,150,150,180,180,621"
820078733 2018-01-12 23:28:39 56237369 12406635 "User:Editingaccount1994/sandbox" 2 FALSE 27782 FALSE 531dizmmloyxffbkdr5vph7owh921eg TRUE "SporkBot" FALSE "798,150,150,150,621,137,137,150,150,350,195,350,195,180,180,350,195,300,150,150,150,180,180,621"
820177382 2018-01-13 13:45:33 56237369 13791031 "User:Editingaccount1994/sandbox" 2 FALSE 27757 FALSE nik9p2u2fuk4yazjxt8ymbicxv5qid9 FALSE "Frietjes" FALSE "798,150,150,150,621,100,621"
822038928 2018-01-24 01:35:22 56237369 2304267 "User:Editingaccount1994/sandbox" 2 FALSE 27667 FALSE gwk6pampl8si1v5pv3kwgteg710sfw3 FALSE "CommonsDelinker" FALSE "798,150,150,150,621,100,621"
819091874 2018-01-07 10:42:20 56237370 1368779 "Anita del Rey" 0 FALSE 25 FALSE n4ozbsgle13p9yywtfrz982ccj8woc9 FALSE "PamD" FALSE
819091883 2018-01-07 10:42:27 56237371 13286072 "User talk:119.94.96.157" 3 FALSE 1274 FALSE ksohnvsbeuzwpl5vb8a3v8m18hva0a7 FALSE "ClueBot NG" FALSE "119,157,119,157,119,157,119,157"
819091914 2018-01-07 10:42:50 56237372 677153 "Category:Ohmi Railway" 14 FALSE 113 FALSE je7aw21fedbwyqsyofpisdrynsu7olr FALSE "Underbar dk" FALSE
819091968 2018-01-07 10:43:32 56237375 882433 "User talk:92.226.219.222" 3 FALSE 199 FALSE cpm4tkzcx4hc6irr9ukbi06ogud8dtq FALSE "TastyPoutine" FALSE
819094036 2018-01-07 11:10:24 56237375 7611264 "User talk:92.226.219.222" 3 FALSE 1840 FALSE artmfz8b2gxhb3pp8a5p4ksplxqfkpg TRUE "AnomieBOT" FALSE
819112363 2018-01-07 14:33:36 56237375 702940 "User talk:92.226.219.222" 3 FALSE 2949 FALSE dn9wj0n8d8pdd5lqe56uw5xamupowr1 FALSE "Only" FALSE "126,126,126,126"
819092004 2018-01-07 10:44:01 56237376 32794237 "User:Dipayanacharya" 2 FALSE 28 FALSE ofueugwatmmn7u73isw732neuza57gk FALSE "Dipayanacharya" FALSE
819092390 2018-01-07 10:49:08 56237376 32794237 "User:Dipayanacharya" 2 FALSE 38 FALSE dsz55xv96ec2uv6w9c1z7c52ipfovbw FALSE "Dipayanacharya" FALSE
819092066 2018-01-07 10:44:56 56237378 21516552 "BSCIC" 0 FALSE 65 FALSE 9ma38hak0ef1ew4fpiutxpnzd8oz1wd FALSE "Vinegarymass911" FALSE
819092102 2018-01-07 10:45:21 56237379 754619 "Category:Women government ministers of Yemen" 14 FALSE 285 FALSE 4dvakoat58bzyf5hmtthxukt29hip6n FALSE "BrownHairedGirl" FALSE
819092135 2018-01-07 10:45:54 56237381 410898 "Talk:List of Morning Glories Characters" 1 FALSE 103 FALSE 2sjrxsc7os9k9pg4su2t4rk2j8nn0h7 FALSE "PRehse" FALSE
819092138 2018-01-07 10:45:56 56237382 13286072 "User talk:106.207.126.114" 3 FALSE 1330 FALSE 3y9t5wpk6ur5jhone75rhm4wjf01fgi FALSE "ClueBot NG" FALSE "106,207,126,114,106,207,126,114,106,207,126,114,106,207,126,114"
819092495 2018-01-07 10:50:22 56237382 31190506 "User talk:106.207.126.114" 3 FALSE 2355 FALSE 8wvn6vh3isyt0dorpe89lztrburgupe FALSE "HindWIKI" FALSE "106,207,126,114,106,207,126,114,106,207,126,114,106,207,126,114"
1 revid date_time articleid editorid title namespace deleted text_chars revert reverteds sha1 minor editor anon threedigits
2 819091731 2018-01-07 10:40:58 56237363 3742946 User talk:86.139.142.254 3 FALSE 1141 FALSE 135nz8q6lfam6cojla7azb7k5alx3t3 FALSE NinjaRobotPirate FALSE 126, 126, 126, 126 126,126,126,126
3 819091755 2018-01-07 10:41:10 56237364 32792125 User talk:Kavin kavitha 3 FALSE 663 FALSE 0pwezjc6yopz0smc8al6ogc4fax5bwo FALSE Kavin kavitha FALSE
4 819091788 2018-01-07 10:41:26 56237365 32621254 User talk:Dr.vivek163 3 FALSE 399 FALSE sz3t2ap7z8bpkdvdvi195f3i35949bv FALSE Amicable always FALSE
5 819091796 2018-01-07 10:41:31 56237366 13286072 User talk:Twistorl 3 FALSE 1260 FALSE r6s5j8j3iykenrhuhpnkpsmmd71vubf FALSE ClueBot NG FALSE
6 819091825 2018-01-07 10:41:51 56237368 8409334 Kom Firin 0 FALSE 2249 FALSE tf5qz2yaswx61zrlm9ovxzuhl7r2dc4 FALSE Khruner FALSE 119, 978, 500, 292, 225, 199, 292 119,978,500,292,225,199,292
7 822610647 2018-01-27 12:16:02 56237368 8409334 Kom Firin 0 FALSE 2230 FALSE e6oa4g0qv64icdaq26uu1zzbyr5hcbh TRUE Khruner FALSE 119, 978, 500, 292, 225, 199, 292 119,978,500,292,225,199,292
8 819091844 2018-01-07 10:42:05 56237369 32794215 User:Editingaccount1994/sandbox 2 FALSE 27840 FALSE 0fyvyh2a8xu41gt8obr34oba0bfixj6 FALSE Editingaccount1994 FALSE 798, 150, 150, 150, 621, 137, 137, 150, 150, 350, 195, 350, 195, 180, 180, 350, 195, 300, 150, 150, 150, 180, 180, 621 798,150,150,150,621,137,137,150,150,350,195,350,195,180,180,350,195,300,150,150,150,180,180,621
9 819093984 2018-01-07 11:09:52 56237369 7611264 User:Editingaccount1994/sandbox 2 FALSE 27787 FALSE 8gy52aolt5rg3eaketwj5v7eiw0apv2 TRUE AnomieBOT FALSE 798, 150, 150, 150, 621, 137, 137, 150, 150, 350, 195, 350, 195, 180, 180, 350, 195, 300, 150, 150, 150, 180, 180, 621 798,150,150,150,621,137,137,150,150,350,195,350,195,180,180,350,195,300,150,150,150,180,180,621
10 820064189 2018-01-12 21:45:50 56237369 12406635 User:Editingaccount1994/sandbox 2 FALSE 27784 FALSE he8ydemaanxlrpftqxkez8jfpge1fsj TRUE SporkBot FALSE 798, 150, 150, 150, 621, 137, 137, 150, 150, 350, 195, 350, 195, 180, 180, 350, 195, 300, 150, 150, 150, 180, 180, 621 798,150,150,150,621,137,137,150,150,350,195,350,195,180,180,350,195,300,150,150,150,180,180,621
11 820078679 2018-01-12 23:28:11 56237369 12406635 User:Editingaccount1994/sandbox 2 FALSE 27783 FALSE 0to17w9rth3url8n7gvucdtobybdq5h TRUE SporkBot FALSE 798, 150, 150, 150, 621, 137, 137, 150, 150, 350, 195, 350, 195, 180, 180, 350, 195, 300, 150, 150, 150, 180, 180, 621 798,150,150,150,621,137,137,150,150,350,195,350,195,180,180,350,195,300,150,150,150,180,180,621
12 820078733 2018-01-12 23:28:39 56237369 12406635 User:Editingaccount1994/sandbox 2 FALSE 27782 FALSE 531dizmmloyxffbkdr5vph7owh921eg TRUE SporkBot FALSE 798, 150, 150, 150, 621, 137, 137, 150, 150, 350, 195, 350, 195, 180, 180, 350, 195, 300, 150, 150, 150, 180, 180, 621 798,150,150,150,621,137,137,150,150,350,195,350,195,180,180,350,195,300,150,150,150,180,180,621
13 820177382 2018-01-13 13:45:33 56237369 13791031 User:Editingaccount1994/sandbox 2 FALSE 27757 FALSE nik9p2u2fuk4yazjxt8ymbicxv5qid9 FALSE Frietjes FALSE 798, 150, 150, 150, 621, 100, 621 798,150,150,150,621,100,621
14 822038928 2018-01-24 01:35:22 56237369 2304267 User:Editingaccount1994/sandbox 2 FALSE 27667 FALSE gwk6pampl8si1v5pv3kwgteg710sfw3 FALSE CommonsDelinker FALSE 798, 150, 150, 150, 621, 100, 621 798,150,150,150,621,100,621
15 819091874 2018-01-07 10:42:20 56237370 1368779 Anita del Rey 0 FALSE 25 FALSE n4ozbsgle13p9yywtfrz982ccj8woc9 FALSE PamD FALSE
16 819091883 2018-01-07 10:42:27 56237371 13286072 User talk:119.94.96.157 3 FALSE 1274 FALSE ksohnvsbeuzwpl5vb8a3v8m18hva0a7 FALSE ClueBot NG FALSE 119, 157, 119, 157, 119, 157, 119, 157 119,157,119,157,119,157,119,157
17 819091914 2018-01-07 10:42:50 56237372 677153 Category:Ohmi Railway 14 FALSE 113 FALSE je7aw21fedbwyqsyofpisdrynsu7olr FALSE Underbar dk FALSE
18 819091968 2018-01-07 10:43:32 56237375 882433 User talk:92.226.219.222 3 FALSE 199 FALSE cpm4tkzcx4hc6irr9ukbi06ogud8dtq FALSE TastyPoutine FALSE
19 819094036 2018-01-07 11:10:24 56237375 7611264 User talk:92.226.219.222 3 FALSE 1840 FALSE artmfz8b2gxhb3pp8a5p4ksplxqfkpg TRUE AnomieBOT FALSE
20 819112363 2018-01-07 14:33:36 56237375 702940 User talk:92.226.219.222 3 FALSE 2949 FALSE dn9wj0n8d8pdd5lqe56uw5xamupowr1 FALSE Only FALSE 126, 126, 126, 126 126,126,126,126
21 819092004 2018-01-07 10:44:01 56237376 32794237 User:Dipayanacharya 2 FALSE 28 FALSE ofueugwatmmn7u73isw732neuza57gk FALSE Dipayanacharya FALSE
22 819092390 2018-01-07 10:49:08 56237376 32794237 User:Dipayanacharya 2 FALSE 38 FALSE dsz55xv96ec2uv6w9c1z7c52ipfovbw FALSE Dipayanacharya FALSE
23 819092066 2018-01-07 10:44:56 56237378 21516552 BSCIC 0 FALSE 65 FALSE 9ma38hak0ef1ew4fpiutxpnzd8oz1wd FALSE Vinegarymass911 FALSE
24 819092102 2018-01-07 10:45:21 56237379 754619 Category:Women government ministers of Yemen 14 FALSE 285 FALSE 4dvakoat58bzyf5hmtthxukt29hip6n FALSE BrownHairedGirl FALSE
25 819092135 2018-01-07 10:45:54 56237381 410898 Talk:List of Morning Glories Characters 1 FALSE 103 FALSE 2sjrxsc7os9k9pg4su2t4rk2j8nn0h7 FALSE PRehse FALSE
26 819092138 2018-01-07 10:45:56 56237382 13286072 User talk:106.207.126.114 3 FALSE 1330 FALSE 3y9t5wpk6ur5jhone75rhm4wjf01fgi FALSE ClueBot NG FALSE 106, 207, 126, 114, 106, 207, 126, 114, 106, 207, 126, 114, 106, 207, 126, 114 106,207,126,114,106,207,126,114,106,207,126,114,106,207,126,114
27 819092495 2018-01-07 10:50:22 56237382 31190506 User talk:106.207.126.114 3 FALSE 2355 FALSE 8wvn6vh3isyt0dorpe89lztrburgupe FALSE HindWIKI FALSE 106, 207, 126, 114, 106, 207, 126, 114, 106, 207, 126, 114, 106, 207, 126, 114 106,207,126,114,106,207,126,114,106,207,126,114,106,207,126,114